From d198183465658a8c16bf3165f25304497a2a8c19 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 23 Jun 2022 15:15:03 +0000 Subject: [PATCH 001/151] Add back the ability to flatten a LEFT JOIN subquery - previously removed due to ticket [cad1ab4cb7b0fc344]. FossilOrigin-Name: f8fe936ad4f7678f9b26ba6fab41c5df9f4938634cdb3286b0f2dcd1357f919a --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/select.c | 8 +------- src/vdbe.c | 7 +++++-- test/join.test | 17 +++++++++++++++++ 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 8882a0bee8..e7bdf38766 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scorrection\sto\sa\scomment.\s\sNo\scode\schanges. -D 2022-06-23T12:36:56.505 +C Add\sback\sthe\sability\sto\sflatten\sa\sLEFT\sJOIN\ssubquery\s-\spreviously\sremoved\ndue\sto\sticket\s[cad1ab4cb7b0fc344]. +D 2022-06-23T15:15:03.618 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -570,7 +570,7 @@ F src/printf.c 6166a30417b05c5b2f82e1f183f75faa2926ad60531c0b688a57dbc951441a20 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c 1655e44c77c51ebbe82924287528a78bd4a4aaaf34189dbae28d19ccf2ca615c F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 2db8bbf13f57cc9872f9b927546e98f5ea583e9e364182426492a434a994421b +F src/select.c 0e07ed5e7dc93ebdc0b5ed4760e147417c30709a2697f263876357009673eb1c F src/shell.c.in 08e59f1cb9d9b1180aba52861aaada0c95f6ddd210488719684e160a0724c806 F src/sqlite.h.in 172528c287399a34f188154017b7268bf82c6d5b780902e361958d2318c4e37c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -642,7 +642,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 602fe229f32a96ceccae4f40824129669582096f7c355f53dbac156c9fecef23 F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 -F src/vdbe.c d7a10fd2afa527886e930ea39eb4a946206e0e439e6cca7424d7373232183010 +F src/vdbe.c c303875051361dc8f5b5c927c2c843ae9d664fc240be4dbeffe243a5af35d6ab F src/vdbe.h 07641758ca8b4f4c6d81ea667ea167c541e6ece21f5574da11e3d21ec37e2662 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c 602610f1252d59cd69742f78a1e2f6fbae40a4b407f5506a6a7b869b0df08ff2 @@ -1161,7 +1161,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 -F test/join.test 5c7f917aa219a125d1df517d3df384c79f74860fdfb2e48c0a599f1e2b3e9ed8 +F test/join.test 6dacc1c7121cac00d17c720677c5a598c8ef6e16780ea9a24edfb96c9b397137 F test/join2.test 466b07233820f5deee66a6c3bf6e4500c8bbf7b83649e67606f5f649c07928c0 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 @@ -1978,8 +1978,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 83ff1a28e3e7a99fa90d5079897d76529c4256eed859bf7cb98b860fbedfdc5b -R 84b3e295df7fc9b4a1509f0d6ebc58aa +P 5fa00959ebc3a46fe400564d5a30e8c44857cd4f0473fb25e84154019eff3f39 +R 7c9e5bbcdf37f239d91e48a825683444 +T *branch * flatten-left-join +T *sym-flatten-left-join * +T -sym-trunk * U drh -Z 71ed77c77f9d7bcfc9156cb5b9ba5369 +Z b50ebd17a3c7c4abceffd64a112d48b3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b65bad0e2e..65826fbe1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5fa00959ebc3a46fe400564d5a30e8c44857cd4f0473fb25e84154019eff3f39 \ No newline at end of file +f8fe936ad4f7678f9b26ba6fab41c5df9f4938634cdb3286b0f2dcd1357f919a \ No newline at end of file diff --git a/src/select.c b/src/select.c index 6f5e8c82ab..12bca38bf8 100644 --- a/src/select.c +++ b/src/select.c @@ -4068,7 +4068,7 @@ static void renumberCursors( ** (3a) the subquery may not be a join and ** (3b) the FROM clause of the subquery may not contain a virtual ** table and -** (3c) the outer query may not be an aggregate. +** (**) REMOVED. Was: the outer query may not be an aggregate. ** (3d) the outer query may not be DISTINCT. ** See also (26) for restrictions on RIGHT JOIN. ** @@ -4273,16 +4273,10 @@ static int flattenSubquery( ** ** which is not at all the same thing. ** - ** If the subquery is the right operand of a LEFT JOIN, then the outer - ** query cannot be an aggregate. (3c) This is an artifact of the way - ** aggregates are processed - there is no mechanism to determine if - ** the LEFT JOIN table should be all-NULL. - ** ** See also tickets #306, #350, and #3300. */ if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ if( pSubSrc->nSrc>1 /* (3a) */ - || isAgg /* (3c) */ || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ diff --git a/src/vdbe.c b/src/vdbe.c index 7857924a80..b01e6416bf 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2639,11 +2639,14 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** If it is, then set register P3 to NULL and jump immediately to P2. ** If P1 is not on a NULL row, then fall through without making any ** changes. +** +** If P1 is not an open cursor, then this opcode is a no-op. */ case OP_IfNullRow: { /* jump */ + VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( p->apCsr[pOp->p1]!=0 ); - if( p->apCsr[pOp->p1]->nullRow ){ + pC = p->apCsr[pOp->p1]; + if( pC && pC->nullRow ){ sqlite3VdbeMemSetNull(aMem + pOp->p3); goto jump_to_p2; } diff --git a/test/join.test b/test/join.test index 38143c0560..c7d1377f2a 100644 --- a/test/join.test +++ b/test/join.test @@ -763,6 +763,23 @@ do_execsql_test join-14.4 { SELECT * FROM (SELECT 111) LEFT JOIN (SELECT c+222 FROM t1) GROUP BY 1; } {111 {}} do_execsql_test join-14.5 { + SELECT * FROM (SELECT 111 AS x UNION ALL SELECT 222) + LEFT JOIN (SELECT c+333 AS y FROM t1) ON x=y GROUP BY 1; +} {111 {} 222 {}} +do_execsql_test join-14.6 { + SELECT * FROM (SELECT 111 AS x UNION ALL SELECT 111) + LEFT JOIN (SELECT c+333 AS y FROM t1) ON x=y GROUP BY 1; +} {111 {}} +do_execsql_test join-14.7 { + SELECT * FROM (SELECT 111 AS x UNION ALL SELECT 111 UNION ALL SELECT 222) + LEFT JOIN (SELECT c+333 AS y FROM t1) ON x=y GROUP BY 1; +} {111 {} 222 {}} +do_execsql_test join-14.8 { + INSERT INTO t1(c) VALUES(-111); + SELECT * FROM (SELECT 111 AS x UNION ALL SELECT 111 UNION ALL SELECT 222) + LEFT JOIN (SELECT c+333 AS y FROM t1) ON x=y GROUP BY 1; +} {111 {} 222 222} +do_execsql_test join-14.9 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(c PRIMARY KEY) WITHOUT ROWID; SELECT * FROM (SELECT 111) LEFT JOIN (SELECT c+222 FROM t1) GROUP BY 1; From 3a6e0c3f0419779785295d263fe990944962d0a1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 23 Jun 2022 20:56:45 +0000 Subject: [PATCH 002/151] Reintroduce flattener constraint (3c), but this time make it apply only if the outer query holds a GROUP BY, not if the outer query is an aggregate. FossilOrigin-Name: 641dfb9182a6cbadb3c452f5420f896791b7844b794f693443bcd38dca14da35 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/select.c | 3 ++- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e7bdf38766..9ad0a00791 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sback\sthe\sability\sto\sflatten\sa\sLEFT\sJOIN\ssubquery\s-\spreviously\sremoved\ndue\sto\sticket\s[cad1ab4cb7b0fc344]. -D 2022-06-23T15:15:03.618 +C Reintroduce\sflattener\sconstraint\s(3c),\sbut\sthis\stime\smake\sit\sapply\sonly\sif\nthe\souter\squery\sholds\sa\sGROUP\sBY,\snot\sif\sthe\souter\squery\sis\san\saggregate. +D 2022-06-23T20:56:45.105 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -570,7 +570,7 @@ F src/printf.c 6166a30417b05c5b2f82e1f183f75faa2926ad60531c0b688a57dbc951441a20 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c 1655e44c77c51ebbe82924287528a78bd4a4aaaf34189dbae28d19ccf2ca615c F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 0e07ed5e7dc93ebdc0b5ed4760e147417c30709a2697f263876357009673eb1c +F src/select.c 8b62787f6985bcfd802978f9e819c4458dd068fef183b087925a1833e831eccd F src/shell.c.in 08e59f1cb9d9b1180aba52861aaada0c95f6ddd210488719684e160a0724c806 F src/sqlite.h.in 172528c287399a34f188154017b7268bf82c6d5b780902e361958d2318c4e37c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1978,11 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5fa00959ebc3a46fe400564d5a30e8c44857cd4f0473fb25e84154019eff3f39 -R 7c9e5bbcdf37f239d91e48a825683444 -T *branch * flatten-left-join -T *sym-flatten-left-join * -T -sym-trunk * +P f8fe936ad4f7678f9b26ba6fab41c5df9f4938634cdb3286b0f2dcd1357f919a +R 83ca326ff6a26325b2e0f5cb84056d0f U drh -Z b50ebd17a3c7c4abceffd64a112d48b3 +Z 3bf3bb97c304e358370c1c605007c0f4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 65826fbe1b..e3c034aad7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8fe936ad4f7678f9b26ba6fab41c5df9f4938634cdb3286b0f2dcd1357f919a \ No newline at end of file +641dfb9182a6cbadb3c452f5420f896791b7844b794f693443bcd38dca14da35 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 12bca38bf8..d3bbd86bb3 100644 --- a/src/select.c +++ b/src/select.c @@ -4068,7 +4068,7 @@ static void renumberCursors( ** (3a) the subquery may not be a join and ** (3b) the FROM clause of the subquery may not contain a virtual ** table and -** (**) REMOVED. Was: the outer query may not be an aggregate. +** (3c) The outer query may not have a GROUP BY ** (3d) the outer query may not be DISTINCT. ** See also (26) for restrictions on RIGHT JOIN. ** @@ -4279,6 +4279,7 @@ static int flattenSubquery( if( pSubSrc->nSrc>1 /* (3a) */ || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ + || (p->pGroupBy!=0) /* (3c) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ ){ return 0; From 521e0b6cb6e387e982676617611b554779239753 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 23 Jun 2022 22:43:11 +0000 Subject: [PATCH 003/151] New test cases for query flattening with LEFT JOIN. FossilOrigin-Name: 27f68e47320c751e3663507500c1c44f0b7f885f89c678fce6a35b1bc372dd64 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/join.test | 13 +++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9ad0a00791..af6fadc4eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reintroduce\sflattener\sconstraint\s(3c),\sbut\sthis\stime\smake\sit\sapply\sonly\sif\nthe\souter\squery\sholds\sa\sGROUP\sBY,\snot\sif\sthe\souter\squery\sis\san\saggregate. -D 2022-06-23T20:56:45.105 +C New\stest\scases\sfor\squery\sflattening\swith\sLEFT\sJOIN. +D 2022-06-23T22:43:11.183 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1161,7 +1161,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 -F test/join.test 6dacc1c7121cac00d17c720677c5a598c8ef6e16780ea9a24edfb96c9b397137 +F test/join.test 21dbc65ab2476b10ae3ed1dcf64a99fb9a40473caa22f4cec2d6a829933c1442 F test/join2.test 466b07233820f5deee66a6c3bf6e4500c8bbf7b83649e67606f5f649c07928c0 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f8fe936ad4f7678f9b26ba6fab41c5df9f4938634cdb3286b0f2dcd1357f919a -R 83ca326ff6a26325b2e0f5cb84056d0f +P 641dfb9182a6cbadb3c452f5420f896791b7844b794f693443bcd38dca14da35 +R bfe2e11b6b9bf77e961e6114078fa2b3 U drh -Z 3bf3bb97c304e358370c1c605007c0f4 +Z 171414e3a15469d4d57a23b44125d739 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e3c034aad7..1d5ca0384c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -641dfb9182a6cbadb3c452f5420f896791b7844b794f693443bcd38dca14da35 \ No newline at end of file +27f68e47320c751e3663507500c1c44f0b7f885f89c678fce6a35b1bc372dd64 \ No newline at end of file diff --git a/test/join.test b/test/join.test index c7d1377f2a..82dbd447f9 100644 --- a/test/join.test +++ b/test/join.test @@ -762,10 +762,23 @@ do_execsql_test join-14.4 { CREATE TABLE t1(c PRIMARY KEY, a TEXT(10000), b TEXT(10000)); SELECT * FROM (SELECT 111) LEFT JOIN (SELECT c+222 FROM t1) GROUP BY 1; } {111 {}} +do_execsql_test join-14.4b { + SELECT * FROM (SELECT 111) LEFT JOIN (SELECT c+222 FROM t1); +} {111 {}} do_execsql_test join-14.5 { SELECT * FROM (SELECT 111 AS x UNION ALL SELECT 222) LEFT JOIN (SELECT c+333 AS y FROM t1) ON x=y GROUP BY 1; } {111 {} 222 {}} +do_execsql_test join-14.5b { + SELECT count(*) + FROM (SELECT 111 AS x UNION ALL SELECT 222) + LEFT JOIN (SELECT c+333 AS y FROM t1) ON x=y; +} {2} +do_execsql_test join-14.5c { + SELECT count(*) + FROM (SELECT c+333 AS y FROM t1) + RIGHT JOIN (SELECT 111 AS x UNION ALL SELECT 222) ON x=y; +} {2} do_execsql_test join-14.6 { SELECT * FROM (SELECT 111 AS x UNION ALL SELECT 111) LEFT JOIN (SELECT c+333 AS y FROM t1) ON x=y GROUP BY 1; From d45ac3947de2d3b5c3e84667bac22a044c72fa64 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 25 Jun 2022 02:37:57 +0000 Subject: [PATCH 004/151] wasm binding: consolidated the two sqlite3_prepare_v2() bindings behind a single dispathcer. Various internal cleanups and refactoring. Branched because trunk is in pencils-down mode for pending 3.39 release. FossilOrigin-Name: ab3e50dab4d71557ab5d179bbd6caf7fb61ab7c51dffc8e4714441c189ce3e5c --- ext/fiddle/sqlite3-api.js | 239 +++++++++++++++++++++++++++++--------- ext/fiddle/testing1.js | 8 +- manifest | 19 +-- manifest.uuid | 2 +- 4 files changed, 202 insertions(+), 66 deletions(-) diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js index 5057eb53f8..3ee57f6818 100644 --- a/ext/fiddle/sqlite3-api.js +++ b/ext/fiddle/sqlite3-api.js @@ -166,9 +166,29 @@ Module.postRun.push(function(namespace/*the module object, the target for SQLITE_INNOCUOUS: 0x000200000, /* sqlite encodings, used for creating UDFs, noting that we will only support UTF8. */ - SQLITE_UTF8: 1 + SQLITE_UTF8: 1, + /* Values for the final argument of sqlite3_result_blob(), + noting that these are interpreted in WASM as pointer + values. */ + SQLITE_TRANSIENT: -1, + SQLITE_STATIC: 0, + + /** + Holds state which are specific to the WASM-related + infrastructure and glue code. It is not expected that client + code will normally need these, but they're exposed here in case it + does. + */ + wasm: { + /** + Proxy for SQM.allocate(). TODO: remove deprecated allocate(), + use _malloc(). The kicker is that allocate() uses + module-init-internal state which isn't normally visible to + us. + */ + allocate: (slab, allocator=SQM.ALLOC_NORMAL)=>SQM.allocate(slab, allocator) + } }; - const cwrap = SQM.cwrap; [/* C-side functions to bind. Each entry is an array with 3 or 4 elements: @@ -219,14 +239,8 @@ Module.postRun.push(function(namespace/*the module object, the target for ["sqlite3_open", "number", ["string", "number"]], //["sqlite3_open_v2", "number", ["string", "number", "number", "string"]], //^^^^ TODO: add the flags needed for the 3rd arg - ["sqlite3_prepare_v2", "number", ["number", "string", "number", "number", "number"]], - ["sqlite3_prepare_v2_sqlptr", "sqlite3_prepare_v2", - /* Impl which requires that the 2nd argument be a pointer to - the SQL string, instead of being converted to a - string. This is used for cases where we require a non-NULL - value for the final argument (exec()'ing multiple - statements from one input string). */ - "number", ["number", "number", "number", "number", "number"]], + /* sqlite3_prepare_v2() is handled separately due to us requiring two + different sets of semantics for that function. */ ["sqlite3_reset", "number", ["number"]], ["sqlite3_result_blob",null,["number", "number", "number", "number"]], ["sqlite3_result_double",null,["number", "number"]], @@ -245,7 +259,104 @@ Module.postRun.push(function(namespace/*the module object, the target for //["sqlite3_normalized_sql", "string", ["number"]] ].forEach(function(a){ const k = (4==a.length) ? a.shift() : a[0]; - api[k] = cwrap.apply(this, a); + api[k] = SQM.cwrap.apply(this, a); + }); + + /** + Proxies for variants of sqlite3_prepare_v2() which have + differing JS/WASM binding semantics. + */ + const prepareMethods = { + /** + This binding expects a JS string as its 2nd argument and + null as its final argument. In order to compile multiple + statements from a single string, the "full" impl (see + below) must be used. + */ + basic: SQM.cwrap('sqlite3_prepare_v2', + "number", ["number", "string", "number"/*MUST always be negative*/, + "number", "number"/*MUST be 0 or null or undefined!*/]), + /* Impl which requires that the 2nd argument be a pointer to + the SQL string, instead of being converted to a + string. This variant is necessary for cases where we + require a non-NULL value for the final argument + (exec()'ing multiple statements from one input + string). For simpler cases, where only the first statement + in the SQL string is required, the wrapper named + sqlite3_prepare_v2() is sufficient and easier to use + because it doesn't require dealing with pointers. + + TODO: hide both of these methods behind a single hand-written + sqlite3_prepare_v2() wrapper which dispatches to the appropriate impl. + */ + full: SQM.cwrap('sqlite3_prepare_v2', + "number", ["number", "number", "number"/*MUST always be negative*/, + "number", "number"]), + }; + + const uint8ToString = (str)=>new TextDecoder('utf-8').decode(str); + //const stringToUint8 = (sql)=>new TextEncoder('utf-8').encode(sql); + + /** + sqlite3_prepare_v2() binding which handles two different uses + with differing JS/WASM semantics: + + 1) sqlite3_prepare_v2(pDb, sqlString, -1, ppStmt [, null]) + + 2) sqlite3_prepare_v2(pDb, sqlPointer, -1, ppStmt, sqlPointerToPointer) + + Note that the SQL length argument (the 3rd argument) must + always be negative because it must be a byte length and that + value is expensive to calculate from JS (where we get the + character length of strings). It is retained in this API's + interface for code/documentation compatibility reasons but is + currently _always_ ignored. When using the 2nd form of this + call, it is critical that the custom-allocated string be + terminated with a 0 byte. (Potential TODO: if this value is >0, + assume the caller knows precisely what they're doing and pass + it on as-is. That approach currently seems fraught with peril.) + + In usage (1), the 2nd argument must be of type string or + Uint8Array (which is assumed to hold SQL). If it is, this + function assumes case (1) and calls the underling C function + with: + + (pDb, sql, -1, ppStmt, null) + + If sql is not a string or Uint8Array, it must be a _pointer_ to + a string which was allocated via api.wasm.allocateUTF8OnStack() + or equivalent (TODO: define "or equivalent"). In that case, the + final argument may be 0/null/undefined or must be a pointer to + which the "tail" of the compiled SQL is written, as documented + for the C-side sqlite3_prepare_v2(). In case (2), the + underlying C function is called with: + + (pDb, sql, -1, ppStmt, pzTail) + + It returns its result and compiled statement as documented in + the C API. Fetching the output pointer (4th argument) requires using + api.wasm.getValue(). + */ + api.sqlite3_prepare_v2 = function(pDb, sql, sqlLen, ppStmt, pzTail){ + if(sql instanceof Uint8Array) sql = uint8ToString(sql); + /* ^^^ TODO: confirm whether this conversion is really + necessary or whether passing on the array as-is will work + as if it were a string. */ + switch(typeof sql){ + case 'string': return prepareMethods.basic(pDb, sql, -1, ppStmt, null); + case 'number': return prepareMethods.full(pDb, sql, -1, ppStmt, pzTail); + default: toss("Invalid SQL argument type for sqlite3_prepare_v2()."); + } + }; + + /** Populate api.wasm... */ + ['getValue','setValue', 'stackSave', 'stackRestore', 'stackAlloc', + 'allocateUTF8OnStack', '_malloc', '_free', + 'addFunction', 'removeFunction' + ].forEach(function(m){ + if(undefined === (api.wasm[m] = SQM[m])){ + toss("Internal init error: Module."+m+" not found."); + } }); /* What follows is colloquially known as "OO API #1". It is a @@ -255,8 +366,6 @@ Module.postRun.push(function(namespace/*the module object, the target for the sqlite3 binding if, e.g., the wrapper is in the main thread and the sqlite3 API is in a worker. */ - /** Memory for use in some pointer-to-pointer-passing routines. */ - const pPtrArg = stackAlloc(4); /** Throws a new error, concatenating all args with a space between each. */ const toss = function(){ @@ -322,9 +431,12 @@ Module.postRun.push(function(namespace/*the module object, the target for } FS.createDataFile("/", fn, buffer, true, true); } - setValue(pPtrArg, 0, "i32"); - this.checkRc(api.sqlite3_open(fn, pPtrArg)); - this._pDb = getValue(pPtrArg, "i32"); + const stack = api.wasm.stackSave(); + const ppDb = api.wasm.stackAlloc(4) /* output (sqlite3**) arg */; + api.wasm.setValue(ppDb, 0, "i32"); + try {this.checkRc(api.sqlite3_open(fn, ppDb));} + finally{api.wasm.stackRestore(stack);} + this._pDb = api.wasm.getValue(ppDb, "i32"); this.filename = fn; this._statements = {/*map of open Stmt _pointers_ to Stmt*/}; this._udfs = {/*map of UDF names to wasm function _pointers_*/}; @@ -378,7 +490,7 @@ Module.postRun.push(function(namespace/*the module object, the target for DB.execMulti(). Does the argument processing/validation, throws on error, and returns a new object on success: - { sql: the SQL, obt: optionsObj, cbArg: function} + { sql: the SQL, opt: optionsObj, cbArg: function} cbArg is only set if the opt.callback is set, in which case it's a function which expects to be passed the current Stmt @@ -386,12 +498,13 @@ Module.postRun.push(function(namespace/*the module object, the target for the input arguments. */ const parseExecArgs = function(args){ - const out = {}; + const out = {opt:{}}; switch(args.length){ case 1: if('string'===typeof args[0]){ out.sql = args[0]; - out.opt = {}; + }else if(args[0] instanceof Uint8Array){ + out.sql = args[0]; }else if(args[0] && 'object'===typeof args[0]){ out.opt = args[0]; out.sql = out.opt.sql; @@ -403,7 +516,11 @@ Module.postRun.push(function(namespace/*the module object, the target for break; default: toss("Invalid argument count for exec()."); }; - if('string'!==typeof out.sql) toss("Missing SQL argument."); + if(out.sql instanceof Uint8Array){ + out.sql = uint8ToString(out.sql); + }else if('string'!==typeof out.sql){ + toss("Missing SQL argument."); + } if(out.opt.callback || out.opt.resultRows){ switch((undefined===out.opt.rowMode) ? 'stmt' : out.opt.rowMode) { @@ -453,7 +570,7 @@ Module.postRun.push(function(namespace/*the module object, the target for delete that._statements[k]; if(s && s._pStmt) s.finalize(); }); - Object.values(this._udfs).forEach(SQM.removeFunction); + Object.values(this._udfs).forEach(api.wasm.removeFunction); delete this._udfs; delete this._statements; api.sqlite3_close_v2(this._pDb); @@ -481,13 +598,21 @@ Module.postRun.push(function(namespace/*the module object, the target for /** Compiles the given SQL and returns a prepared Stmt. This is the only way to create new Stmt objects. Throws on error. + + The given SQL must be a string, a Uint8Array holding SQL, + or a WASM pointer to memory allocated using + api.wasm.allocateUTF8OnStack() (or equivalent (a term which + is yet to be defined precisely)). */ prepare: function(sql){ affirmDbOpen(this); - setValue(pPtrArg,0,"i32"); - this.checkRc(api.sqlite3_prepare_v2(this._pDb, sql, -1, pPtrArg, null)); - const pStmt = getValue(pPtrArg, "i32"); - if(!pStmt) toss("Empty SQL is not permitted."); + const stack = api.wasm.stackSave(); + const ppStmt = api.wasm.stackAlloc(4)/* output (sqlite3_stmt**) arg */; + api.wasm.setValue(ppStmt, 0, "i32"); + try {this.checkRc(api.sqlite3_prepare_v2(this._pDb, sql, -1, ppStmt, null));} + finally {api.wasm.stackRestore(stack);} + const pStmt = api.wasm.getValue(ppStmt, "i32"); + if(!pStmt) toss("Cannot prepare empty SQL."); const stmt = new Stmt(this, pStmt, BindTypes); this._statements[pStmt] = stmt; return stmt; @@ -585,7 +710,7 @@ Module.postRun.push(function(namespace/*the module object, the target for properties: - .sql = the SQL to run (unless it's provided as the first - argument). + argument). This must be of type string or Uint8Array. - .bind = a single value valid as an argument for Stmt.bind(). This is ONLY applied to the FIRST non-empty @@ -638,23 +763,27 @@ Module.postRun.push(function(namespace/*the module object, the target for ? arguments[0] : parseExecArgs(arguments)); if(!arg.sql) return this; const opt = arg.opt; - const stack = stackSave(); + const stack = api.wasm.stackSave(); let stmt; let bind = opt.bind; let rowMode = ( (opt.callback && opt.rowMode) ? opt.rowMode : false); try{ - let pSql = SQM.allocateUTF8OnStack(arg.sql) - const pzTail = stackAlloc(4); - while(getValue(pSql, "i8")){ - setValue(pPtrArg, 0, "i32"); - setValue(pzTail, 0, "i32"); - this.checkRc(api.sqlite3_prepare_v2_sqlptr( - this._pDb, pSql, -1, pPtrArg, pzTail + const sql = (arg.sql instanceof Uint8Array) + ? uint8ToString(arg.sql) + : arg.sql; + let pSql = api.wasm.allocateUTF8OnStack(sql) + const ppStmt = api.wasm.stackAlloc(8) /* output (sqlite3_stmt**) arg */; + const pzTail = ppStmt + 4 /* final arg to sqlite3_prepare_v2_sqlptr() */; + while(api.wasm.getValue(pSql, "i8")){ + api.wasm.setValue(ppStmt, 0, "i32"); + api.wasm.setValue(pzTail, 0, "i32"); + this.checkRc(api.sqlite3_prepare_v2( + this._pDb, pSql, -1, ppStmt, pzTail )); - const pStmt = getValue(pPtrArg, "i32"); - pSql = getValue(pzTail, "i32"); + const pStmt = api.wasm.getValue(ppStmt, "i32"); + pSql = api.wasm.getValue(pzTail, "i32"); if(!pStmt) continue; if(opt.saveSql){ opt.saveSql.push(api.sqlite3_sql(pStmt).trim()); @@ -683,7 +812,7 @@ Module.postRun.push(function(namespace/*the module object, the target for delete stmt._isLocked; stmt.finalize(); } - stackRestore(stack); + api.wasm.stackRestore(stack); } return this; }/*execMulti()*/, @@ -736,7 +865,6 @@ Module.postRun.push(function(namespace/*the module object, the target for - .directOnly = SQLITE_DIRECTONLY - .innocuous = SQLITE_INNOCUOUS - Maintenance reminder: the ability to add new WASM-accessible functions to the runtime requires that the WASM build is compiled with emcc's `-sALLOW_TABLE_GROWTH` @@ -769,7 +897,7 @@ Module.postRun.push(function(namespace/*the module object, the target for let i, pVal, valType, arg; const tgt = []; for(i = 0; i < argc; ++i){ - pVal = getValue(pArgv + (4 * i), "i32"); + pVal = api.wasm.getValue(pArgv + (4 * i), "i32"); valType = api.sqlite3_value_type(pVal); switch(valType){ case api.SQLITE_INTEGER: @@ -806,8 +934,7 @@ Module.postRun.push(function(namespace/*the module object, the target for break; } case 'string': - api.sqlite3_result_text(pCx, val, -1, - -1/*==SQLITE_TRANSIENT*/); + api.sqlite3_result_text(pCx, val, -1, api.SQLITE_TRANSIENT); break; case 'object': if(null===val) { @@ -815,9 +942,10 @@ Module.postRun.push(function(namespace/*the module object, the target for break; }else if(undefined!==val.length){ const pBlob = - SQM.allocate(val, SQM.ALLOC_NORMAL); - api.sqlite3_result_blob(pCx, pBlob, val.length, -1/*==SQLITE_TRANSIENT*/); - SQM._free(blobptr); + api.wasm.allocate(val); + api.sqlite3_result_blob(pCx, pBlob, val.length, + api.SQLITE_TRANSIENT); + api.wasm._free(blobptr); break; } // else fall through @@ -833,7 +961,7 @@ Module.postRun.push(function(namespace/*the module object, the target for api.sqlite3_result_error(pCx, e.message, -1); } }; - const pUdf = SQM.addFunction(wrapper, "viii"); + const pUdf = api.wasm.addFunction(wrapper, "viii"); let fFlags = 0; if(getOwnOption(opt, 'deterministic')) fFlags |= api.SQLITE_DETERMINISTIC; if(getOwnOption(opt, 'directOnly')) fFlags |= api.SQLITE_DIRECTONLY; @@ -846,11 +974,11 @@ Module.postRun.push(function(namespace/*the module object, the target for api.SQLITE_UTF8 | fFlags, null/*pApp*/, pUdf, null/*xStep*/, null/*xFinal*/, null/*xDestroy*/)); }catch(e){ - SQM.removeFunction(pUdf); + api.wasm.removeFunction(pUdf); throw e; } if(this._udfs.hasOwnProperty(name)){ - SQM.removeFunction(this._udfs[name]); + api.wasm.removeFunction(this._udfs[name]); } this._udfs[name] = pUdf; return this; @@ -999,7 +1127,7 @@ Module.postRun.push(function(namespace/*the module object, the target for f._ = { string: function(stmt, ndx, val, asBlob){ const bytes = intArrayFromString(val,true); - const pStr = SQM.allocate(bytes, ALLOC_NORMAL); + const pStr = api.wasm.allocate(bytes); stmt._allocs.push(pStr); const func = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; return func(stmt._pStmt, ndx, pStr, bytes.length, 0); @@ -1038,7 +1166,7 @@ Module.postRun.push(function(namespace/*the module object, the target for toss("Binding a value as a blob requires", "that it have a length member."); } - const pBlob = SQM.allocate(val, ALLOC_NORMAL); + const pBlob = api.wasm.allocate(val); stmt._allocs.push(pBlob); rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, len, 0); } @@ -1054,7 +1182,7 @@ Module.postRun.push(function(namespace/*the module object, the target for const freeBindMemory = function(stmt){ let m; while(undefined !== (m = stmt._allocs.pop())){ - SQM._free(m); + api.wasm._free(m); } return stmt; }; @@ -1481,7 +1609,7 @@ Module.postRun.push(function(namespace/*the module object, the target for if(self === self.window){ /* This is running in the main window thread, so we're done. */ - setTimeout(()=>postMessage({type:'sqlite3-api',data:'loaded'}), 0); + postMessage({type:'sqlite3-api',data:'loaded'}); return; } /****************************************************************** @@ -1489,14 +1617,15 @@ Module.postRun.push(function(namespace/*the module object, the target for in Worker threads. ******************************************************************/ - /* + /** UNDER CONSTRUCTION We need an API which can proxy the DB API via a Worker message interface. The primary quirky factor in such an API is that we cannot pass callback functions between the window thread and a worker thread, so we have to receive all db results via - asynchronous message-passing. + asynchronous message-passing. That requires an asychronous API + with a distinctly different shape that the main OO API. Certain important considerations here include: @@ -1777,5 +1906,5 @@ Module.postRun.push(function(namespace/*the module object, the target for wState.post(evType, response, wMsgHandler.xfer); }; - setTimeout(()=>postMessage({type:'sqlite3-api',data:'loaded'}), 0); -}); + postMessage({type:'sqlite3-api',data:'loaded'}); +})/*postRun.push(...)*/; diff --git a/ext/fiddle/testing1.js b/ext/fiddle/testing1.js index a6f0062dcc..ceea0e6200 100644 --- a/ext/fiddle/testing1.js +++ b/ext/fiddle/testing1.js @@ -27,7 +27,10 @@ const api = sqlite3.api; log("Basic sanity tests..."); T.assert(db._pDb); - let st = db.prepare("select 3 as a"); + let st = db.prepare( + new TextEncoder('utf-8').encode("select 3 as a") + /* Testing handling of Uint8Array input */ + ); //log("statement =",st); T.assert(st._pStmt) .assert(!st._mayGet) @@ -47,6 +50,7 @@ .assert(3.0 === st.get(0,api.SQLITE_FLOAT)) .assert(3.0 === st.getFloat(0)) .assert(st.get(0,api.SQLITE_BLOB) instanceof Uint8Array) + .assert(1===st.get(0,api.SQLITE_BLOB).length) .assert(st.getBlob(0) instanceof Uint8Array) .assert(3 === st.get([])[0]) .assert(3 === st.get({}).a) @@ -72,7 +76,7 @@ INSERT INTO t(a,b) VALUES(1,2),(3,4),(?,?);`, //log("Exec'd SQL:", list); let counter = 0, colNames = []; list.length = 0; - db.exec("SELECT a a, b b FROM t",{ + db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ rowMode: 'object', resultRows: list, columnNames: colNames, diff --git a/manifest b/manifest index 621dd03662..dffd8233a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\sUBSAN\swarning\sassociated\swith\sPRAGMA\sschema_version\nfound\sby\sOSSFuzz. -D 2022-06-24T12:56:48.588 +C wasm\sbinding:\sconsolidated\sthe\stwo\ssqlite3_prepare_v2()\sbindings\sbehind\sa\ssingle\sdispathcer.\sVarious\sinternal\scleanups\sand\srefactoring.\sBranched\sbecause\strunk\sis\sin\spencils-down\smode\sfor\spending\s3.39\srelease. +D 2022-06-25T02:37:57.122 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -65,11 +65,11 @@ F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba83 F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf -F ext/fiddle/sqlite3-api.js ccf4bd0c1c5bbb3be3469573423d6c53991941bec497eac63e9f17ea13bf8952 +F ext/fiddle/sqlite3-api.js aff4fe5583d02cb7207cc27a92d2cc708fca8a0eba14339f519059298f6bbc5e F ext/fiddle/sqlite3-worker.js a9c2b614beca187dbdd8c053ec2770cc61ec1ac9c0ec6398ceb49a79f705a421 F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a -F ext/fiddle/testing1.js e2fa02ac8adbd21c69bc50cfcb79bfc26af0d30a8d6b95ac473a17e0dc9de733 +F ext/fiddle/testing1.js 2ba8d14b335cdfc694757c25a3ffcfa76f4696bb21187db5c12ea6e505c44af0 F ext/fiddle/testing2.html 9063b2430ade2fe9da4e711addd1b51a2741cf0c7ebf6926472a5e5dd63c0bc4 F ext/fiddle/testing2.js 7b45b4e7fddbd51dbaf89b6722c02758051b34bac5a98c11b569a7e7572f88ee F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e @@ -1978,8 +1978,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db5266dec601a9513bc8dd09a9f8bb4aef55b780d22610946099e8edd4836587 -R 09efcead2b1be4144ef279312bf3e194 -U drh -Z 1fc91f4cca35906afdee86eebc313610 +P e93fd170ce4ae91d572c46d03f68f55d00091d0188030517455017d90d212587 +R 3edce7228a2d729bf332a89ed3af6305 +T *branch * wasm-cleanups +T *sym-wasm-cleanups * +T -sym-trunk * Cancelled\sby\sbranch. +U stephan +Z e1adc8db1d84e5f34db3da1c68e91f04 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1a1f800066..e24c1ef14b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e93fd170ce4ae91d572c46d03f68f55d00091d0188030517455017d90d212587 \ No newline at end of file +ab3e50dab4d71557ab5d179bbd6caf7fb61ab7c51dffc8e4714441c189ce3e5c \ No newline at end of file From af260308b3bd433f759644cbb81609f8e54905e9 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 25 Jun 2022 02:54:20 +0000 Subject: [PATCH 005/151] Minor wasm doc tweaks. FossilOrigin-Name: 42dc500819bfc1308a9542aa2cae4f6dfd98a29237c59cec82e0e6f9e0bf3779 --- ext/fiddle/sqlite3-api.js | 19 +++++++++++-------- manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js index 3ee57f6818..74f3e35480 100644 --- a/ext/fiddle/sqlite3-api.js +++ b/ext/fiddle/sqlite3-api.js @@ -321,7 +321,13 @@ Module.postRun.push(function(namespace/*the module object, the target for function assumes case (1) and calls the underling C function with: - (pDb, sql, -1, ppStmt, null) + (pDb, sqlAsString, -1, ppStmt, null) + + The pzTail argument is ignored in this case because its result + is meaningless when a string-type value is passed through + (because the string goes through another level of internal + conversion for WASM's sake and the result pointer would refer + to that conversion's memory, not the passed-in string). If sql is not a string or Uint8Array, it must be a _pointer_ to a string which was allocated via api.wasm.allocateUTF8OnStack() @@ -331,17 +337,14 @@ Module.postRun.push(function(namespace/*the module object, the target for for the C-side sqlite3_prepare_v2(). In case (2), the underlying C function is called with: - (pDb, sql, -1, ppStmt, pzTail) + (pDb, sqlAsPointer, -1, ppStmt, pzTail) It returns its result and compiled statement as documented in - the C API. Fetching the output pointer (4th argument) requires using - api.wasm.getValue(). + the C API. Fetching the output pointers (4th and 5th + parameters) requires using api.wasm.getValue(). */ api.sqlite3_prepare_v2 = function(pDb, sql, sqlLen, ppStmt, pzTail){ if(sql instanceof Uint8Array) sql = uint8ToString(sql); - /* ^^^ TODO: confirm whether this conversion is really - necessary or whether passing on the array as-is will work - as if it were a string. */ switch(typeof sql){ case 'string': return prepareMethods.basic(pDb, sql, -1, ppStmt, null); case 'number': return prepareMethods.full(pDb, sql, -1, ppStmt, pzTail); @@ -349,7 +352,7 @@ Module.postRun.push(function(namespace/*the module object, the target for } }; - /** Populate api.wasm... */ + /** Populate api.wasm with several members of the module object... */ ['getValue','setValue', 'stackSave', 'stackRestore', 'stackAlloc', 'allocateUTF8OnStack', '_malloc', '_free', 'addFunction', 'removeFunction' diff --git a/manifest b/manifest index dffd8233a8..6a4922ca36 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm\sbinding:\sconsolidated\sthe\stwo\ssqlite3_prepare_v2()\sbindings\sbehind\sa\ssingle\sdispathcer.\sVarious\sinternal\scleanups\sand\srefactoring.\sBranched\sbecause\strunk\sis\sin\spencils-down\smode\sfor\spending\s3.39\srelease. -D 2022-06-25T02:37:57.122 +C Minor\swasm\sdoc\stweaks. +D 2022-06-25T02:54:20.783 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -65,7 +65,7 @@ F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba83 F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf -F ext/fiddle/sqlite3-api.js aff4fe5583d02cb7207cc27a92d2cc708fca8a0eba14339f519059298f6bbc5e +F ext/fiddle/sqlite3-api.js c54f0ee6a65a59284e6b0184f556376b35f5996b6b82607d7275a95f759a380a F ext/fiddle/sqlite3-worker.js a9c2b614beca187dbdd8c053ec2770cc61ec1ac9c0ec6398ceb49a79f705a421 F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a @@ -1978,11 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e93fd170ce4ae91d572c46d03f68f55d00091d0188030517455017d90d212587 -R 3edce7228a2d729bf332a89ed3af6305 -T *branch * wasm-cleanups -T *sym-wasm-cleanups * -T -sym-trunk * Cancelled\sby\sbranch. +P ab3e50dab4d71557ab5d179bbd6caf7fb61ab7c51dffc8e4714441c189ce3e5c +R bbdb5fff926c789b277af363fc1aa00d U stephan -Z e1adc8db1d84e5f34db3da1c68e91f04 +Z 54385a773b385e5557f29a3390198b0f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e24c1ef14b..66b7720610 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab3e50dab4d71557ab5d179bbd6caf7fb61ab7c51dffc8e4714441c189ce3e5c \ No newline at end of file +42dc500819bfc1308a9542aa2cae4f6dfd98a29237c59cec82e0e6f9e0bf3779 \ No newline at end of file From 766ba7917e0f01929fa69b9d099555b9ca0b8b69 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 25 Jun 2022 03:53:43 +0000 Subject: [PATCH 006/151] wasm: corrected the propagation of text/blob values via UDFs. DB.exec()'s sql may now be an array of strings which get concatenated together before passing it on to sqlite3_prepare_v2(). DB.exec()'s callback now applies to the first statement which has result columns instead of only the first statement. Fixed a precedence but which caused isInt32() to report false positives. FossilOrigin-Name: 37a8fecb56793fbd3763a2240a0bd62b639c811934d3af2ef0e5ff579073d632 --- ext/fiddle/sqlite3-api.js | 47 ++++++++++++++++++++++++--------------- ext/fiddle/testing1.js | 36 +++++++++++++++++++++--------- manifest | 14 ++++++------ manifest.uuid | 2 +- 4 files changed, 63 insertions(+), 36 deletions(-) diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js index 74f3e35480..f4d0b68d00 100644 --- a/ext/fiddle/sqlite3-api.js +++ b/ext/fiddle/sqlite3-api.js @@ -186,7 +186,12 @@ Module.postRun.push(function(namespace/*the module object, the target for module-init-internal state which isn't normally visible to us. */ - allocate: (slab, allocator=SQM.ALLOC_NORMAL)=>SQM.allocate(slab, allocator) + allocate: (slab, allocator=SQM.ALLOC_NORMAL)=>SQM.allocate(slab, allocator), + /** + The buffer which holds the heap memory managed by the + emscripten-installed allocator. + */ + HEAP8: SQM.HEAP8 } }; [/* C-side functions to bind. Each entry is an array with 3 or 4 @@ -485,7 +490,7 @@ Module.postRun.push(function(namespace/*the module object, the target for /** Returns true if n is a 32-bit (signed) integer, else false. */ const isInt32 = function(n){ - return (n===n|0 && n<0xFFFFFFFF) ? true : undefined; + return (n===(n|0) && n<0xFFFFFFFF) ? true : undefined; }; /** @@ -521,6 +526,8 @@ Module.postRun.push(function(namespace/*the module object, the target for }; if(out.sql instanceof Uint8Array){ out.sql = uint8ToString(out.sql); + }else if(Array.isArray(out.sql)){ + out.sql = out.sql.join(''); }else if('string'!==typeof out.sql){ toss("Missing SQL argument."); } @@ -713,7 +720,10 @@ Module.postRun.push(function(namespace/*the module object, the target for properties: - .sql = the SQL to run (unless it's provided as the first - argument). This must be of type string or Uint8Array. + argument). This must be of type string, Uint8Array, or an + array of strings (in which case they're concatenated + together as-is, with no separator between elements, + before evaluation). - .bind = a single value valid as an argument for Stmt.bind(). This is ONLY applied to the FIRST non-empty @@ -721,12 +731,13 @@ Module.postRun.push(function(namespace/*the module object, the target for parameters. (Empty statements are skipped entirely.) - .callback = a function which gets called for each row of - the FIRST statement in the SQL (if it has any result - rows). The second argument passed to the callback is - always the current Stmt object (so that the caller - may collect column names, or similar). The first - argument passed to the callback defaults to the current - Stmt object but may be changed with ... + the FIRST statement in the SQL which has result + _columns_, but only if that statement has any result + _rows_. The second argument passed to the callback is + always the current Stmt object (so that the caller may + collect column names, or similar). The first argument + passed to the callback defaults to the current Stmt + object but may be changed with ... - .rowMode = a string describing what type of argument should be passed as the first argument to the callback. A @@ -796,7 +807,7 @@ Module.postRun.push(function(namespace/*the module object, the target for stmt.bind(bind); bind = null; } - if(opt.callback && null!==rowMode){ + if(opt.callback && null!==rowMode && stmt.columnCount){ while(stmt.step()){ stmt._isLocked = true; callback(arg.cbArg(stmt), stmt); @@ -907,15 +918,15 @@ Module.postRun.push(function(namespace/*the module object, the target for case api.SQLITE_FLOAT: arg = api.sqlite3_value_double(pVal); break; - case SQLITE_TEXT: + case api.SQLITE_TEXT: arg = api.sqlite3_value_text(pVal); break; - case SQLITE_BLOB:{ - const n = api.sqlite3_value_bytes(ptr); - const pBlob = api.sqlite3_value_blob(ptr); + case api.SQLITE_BLOB:{ + const n = api.sqlite3_value_bytes(pVal); + const pBlob = api.sqlite3_value_blob(pVal); arg = new Uint8Array(n); let i; - for(i = 0; i < n; ++i) arg[i] = HEAP8[pBlob+i]; + for(i = 0; i < n; ++i) arg[i] = api.wasm.HEAP8[pBlob+i]; break; } default: @@ -948,7 +959,7 @@ Module.postRun.push(function(namespace/*the module object, the target for api.wasm.allocate(val); api.sqlite3_result_blob(pCx, pBlob, val.length, api.SQLITE_TRANSIENT); - api.wasm._free(blobptr); + api.wasm._free(pBlob); break; } // else fall through @@ -1082,7 +1093,7 @@ Module.postRun.push(function(namespace/*the module object, the target for const affirmParamIndex = function(stmt,key){ const n = ('number'===typeof key) ? key : api.sqlite3_bind_parameter_index(stmt._pStmt, key); - if(0===n || (n===key && (n!==(n|0)/*floating point*/))){ + if(0===n || !isInt32(n)){ toss("Invalid bind() parameter name: "+key); } else if(n<1 || n>stmt.parameterCount) toss("Bind index",key,"is out of range."); @@ -1456,7 +1467,7 @@ Module.postRun.push(function(namespace/*the module object, the target for const n = api.sqlite3_column_bytes(this._pStmt, ndx); const ptr = api.sqlite3_column_blob(this._pStmt, ndx); const rc = new Uint8Array(n); - for(let i = 0; i < n; ++i) rc[i] = HEAP8[ptr + i]; + for(let i = 0; i < n; ++i) rc[i] = api.wasm.HEAP8[ptr + i]; if(n && this.db._blobXfer instanceof Array){ /* This is an optimization soley for the Worker-based API. These values will be diff --git a/ext/fiddle/testing1.js b/ext/fiddle/testing1.js index ceea0e6200..0b178d8d9e 100644 --- a/ext/fiddle/testing1.js +++ b/ext/fiddle/testing1.js @@ -66,11 +66,14 @@ let list = []; db.exec({ - sql:`CREATE TABLE t(a,b); -INSERT INTO t(a,b) VALUES(1,2),(3,4),(?,?);`, + sql:['CREATE TABLE t(a,b);', + "INSERT INTO t(a,b) VALUES(1,2),(3,4),", + "(?,?),('blob',X'6869');" + ], multi: true, saveSql: list, bind: [5,6] + /* Achtung: ^^^ bind support might be removed from multi-mode exec. */ }); T.assert(2 === list.length); //log("Exec'd SQL:", list); @@ -82,23 +85,24 @@ INSERT INTO t(a,b) VALUES(1,2),(3,4),(?,?);`, columnNames: colNames, callback: function(row,stmt){ ++counter; - T.assert(row.a%2 && row.a<6); + T.assert((row.a%2 && row.a<6) || 'blob'===row.a); } }); T.assert(2 === colNames.length) .assert('a' === colNames[0]) - .assert(3 === counter) - .assert(3 === list.length); + .assert(4 === counter) + .assert(4 === list.length); list.length = 0; db.exec("SELECT a a, b b FROM t",{ rowMode: 'array', callback: function(row,stmt){ ++counter; T.assert(Array.isArray(row)) - .assert(0===row[1]%2 && row[1]<7); + .assert((0===row[1]%2 && row[1]<7) + || (row[1] instanceof Uint8Array)); } }); - T.assert(6 === counter); + T.assert(8 === counter); }; const testUDF = function(db){ @@ -106,7 +110,7 @@ INSERT INTO t(a,b) VALUES(1,2),(3,4),(?,?);`, db.createFunction("foo",function(a,b){return a+b}); T.assert(7===db.selectValue("select foo(3,4)")). assert(5===db.selectValue("select foo(3,?)",2)). - assert(5===db.selectValue("select foo(?,?)",[1,4])). + assert(5===db.selectValue("select foo(?,?2)",[1,4])). assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5})); db.createFunction("bar", { arity: -1, @@ -115,15 +119,26 @@ INSERT INTO t(a,b) VALUES(1,2),(3,4),(?,?);`, for(let i = 0; i < arguments.length; ++i) rc += arguments[i]; return rc; } + }).createFunction({ + name: "asis", + callback: function(arg){ + return arg; + } }); log("Testing DB::selectValue() w/ UDF..."); T.assert(0===db.selectValue("select bar()")). assert(1===db.selectValue("select bar(1)")). assert(3===db.selectValue("select bar(1,2)")). - assert(-1===db.selectValue("select bar(1,2,-4)")); + assert(-1===db.selectValue("select bar(1,2,-4)")). + assert('hi'===db.selectValue("select asis('hi')")); + let blob = db.selectValue("select asis(X'6869')"); + T.assert(blob instanceof Uint8Array). + assert(2 === blob.length). + assert(0x68==blob[0] && 0x69==blob[1]); const eqApprox = function(v1,v2,factor=0.05){ + //log('eqApprox',v1, v2); return v1>=(v2-factor) && v1<=(v2+factor); }; @@ -132,7 +147,8 @@ INSERT INTO t(a,b) VALUES(1,2),(3,4),(?,?);`, assert(null === db.selectValue("select ?",null)). assert(null === db.selectValue("select ?",[null])). assert(null === db.selectValue("select $a",{$a:null})). - assert(eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))) + assert(eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). + assert(eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))) ; }; diff --git a/manifest b/manifest index 6a4922ca36..ab24710ad4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\swasm\sdoc\stweaks. -D 2022-06-25T02:54:20.783 +C wasm:\scorrected\sthe\spropagation\sof\stext/blob\svalues\svia\sUDFs.\sDB.exec()'s\ssql\smay\snow\sbe\san\sarray\sof\sstrings\swhich\sget\sconcatenated\stogether\sbefore\spassing\sit\son\sto\ssqlite3_prepare_v2().\sDB.exec()'s\scallback\snow\sapplies\sto\sthe\sfirst\sstatement\swhich\shas\sresult\scolumns\sinstead\sof\sonly\sthe\sfirst\sstatement.\sFixed\sa\sprecedence\sbut\swhich\scaused\sisInt32()\sto\sreport\sfalse\spositives. +D 2022-06-25T03:53:43.503 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -65,11 +65,11 @@ F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba83 F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf -F ext/fiddle/sqlite3-api.js c54f0ee6a65a59284e6b0184f556376b35f5996b6b82607d7275a95f759a380a +F ext/fiddle/sqlite3-api.js b706be1f777a4508f9e3e237301e11c108933ae651b3fa599bf047cac34b5bdb F ext/fiddle/sqlite3-worker.js a9c2b614beca187dbdd8c053ec2770cc61ec1ac9c0ec6398ceb49a79f705a421 F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a -F ext/fiddle/testing1.js 2ba8d14b335cdfc694757c25a3ffcfa76f4696bb21187db5c12ea6e505c44af0 +F ext/fiddle/testing1.js 3e5f1fb22764ec735396db518b85ea6b3070f6c2da479cba69c8d8e89f8299f8 F ext/fiddle/testing2.html 9063b2430ade2fe9da4e711addd1b51a2741cf0c7ebf6926472a5e5dd63c0bc4 F ext/fiddle/testing2.js 7b45b4e7fddbd51dbaf89b6722c02758051b34bac5a98c11b569a7e7572f88ee F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ab3e50dab4d71557ab5d179bbd6caf7fb61ab7c51dffc8e4714441c189ce3e5c -R bbdb5fff926c789b277af363fc1aa00d +P 42dc500819bfc1308a9542aa2cae4f6dfd98a29237c59cec82e0e6f9e0bf3779 +R a152bf70e7d16ac14f437dceeb13aaae U stephan -Z 54385a773b385e5557f29a3390198b0f +Z 385f37f8ee9972f16da6e6bace96df52 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 66b7720610..cd66bdca9e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42dc500819bfc1308a9542aa2cae4f6dfd98a29237c59cec82e0e6f9e0bf3779 \ No newline at end of file +37a8fecb56793fbd3763a2240a0bd62b639c811934d3af2ef0e5ff579073d632 \ No newline at end of file From f170b0868dd616fbc40946d95fafe02d70ffc4e4 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 25 Jun 2022 06:46:22 +0000 Subject: [PATCH 007/151] wasm: eliminated the dependency on the deprecated emcc-provided allocate() function. Adjacent cleanups in blob binding. FossilOrigin-Name: 140618b212e3aa9ff2df20f22af846a1c4c5ffaeefd84330446f61362b39a8f1 --- ext/fiddle/EXPORTED_RUNTIME_METHODS | 5 +- ext/fiddle/sqlite3-api.js | 138 +++++++++++++++++++--------- ext/fiddle/testing1.js | 37 ++++++-- manifest | 16 ++-- manifest.uuid | 2 +- 5 files changed, 136 insertions(+), 62 deletions(-) diff --git a/ext/fiddle/EXPORTED_RUNTIME_METHODS b/ext/fiddle/EXPORTED_RUNTIME_METHODS index 763722d236..3d06c32192 100644 --- a/ext/fiddle/EXPORTED_RUNTIME_METHODS +++ b/ext/fiddle/EXPORTED_RUNTIME_METHODS @@ -1,12 +1,11 @@ -ALLOC_NORMAL FS -UTF8ToString addFunction -allocate allocateUTF8OnStack ccall cwrap getValue +intArrayFromString +lengthBytesUTF8 removeFunction setValue stackAlloc diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js index f4d0b68d00..eabac6cc5a 100644 --- a/ext/fiddle/sqlite3-api.js +++ b/ext/fiddle/sqlite3-api.js @@ -122,6 +122,27 @@ Module.postRun.push(function(namespace/*the module object, the target for const SQM = namespace/*the sqlite module object */; + /** Throws a new Error, the message of which is the concatenation + all args with a space between each. */ + const toss = function(){ + throw new Error(Array.prototype.join.call(arguments, ' ')); + }; + + /** + Returns true if v appears to be one of our supported TypedArray types: + Uint8Array or Int8Array. + */ + const isSupportedTypedArray = function(v){ + return v && (undefined!==v.byteLength) && (v.byteLength === v.length); + }; + + /** Returns true if isSupportedTypedArray(v) does, else throws with a message + that v is not a supported TypedArray value. */ + const affirmSupportedTypedArray = function(v){ + return isSupportedTypedArray(v) + || toss("Value is not of a supported TypedArray type."); + }; + /** Set up the main sqlite3 binding API here, mimicking the C API as closely as we can. @@ -181,15 +202,38 @@ Module.postRun.push(function(namespace/*the module object, the target for */ wasm: { /** - Proxy for SQM.allocate(). TODO: remove deprecated allocate(), - use _malloc(). The kicker is that allocate() uses - module-init-internal state which isn't normally visible to - us. + api.wasm._malloc()'s srcTypedArray.byteLength bytes, + populates them with the values from the source array, + and returns the pointer to that memory. The pointer + must eventually be passed to api.wasm._free() to clean + it up. + + As a special case, to avoid further special cases where + this is used, if srcTypedArray.byteLength is 0, it + allocates a single byte and sets it to the value 0. + + ACHTUNG: this currently only works for Uint8Array and + Int8Array types. */ - allocate: (slab, allocator=SQM.ALLOC_NORMAL)=>SQM.allocate(slab, allocator), + mallocFromTypedArray: function(srcTypedArray){ + affirmSupportedTypedArray(srcTypedArray); + const pRet = api.wasm._malloc(srcTypedArray.byteLength || 1); + if(srcTypedArray.byteLength){ + api.wasm._malloc.HEAP.set(srcTypedArray, pRet); + /* That unfortunately does not behave intuitively + when copying, e.g., the contents of a + Uint16Array, copying only 1 byte of each + entry instead of blitting the whole array + contents over the destination array. A potential TODO + is handle that copying here so that we can support a wider + array (haha) of bindable-as-blob types. */ + } + else api.wasm._malloc.HEAP[pRet] = 0; + return pRet; + }, /** - The buffer which holds the heap memory managed by the - emscripten-installed allocator. + The TypedArray buffer which holds the heap memory + managed by the emscripten-installed _malloc(). */ HEAP8: SQM.HEAP8 } @@ -360,12 +404,20 @@ Module.postRun.push(function(namespace/*the module object, the target for /** Populate api.wasm with several members of the module object... */ ['getValue','setValue', 'stackSave', 'stackRestore', 'stackAlloc', 'allocateUTF8OnStack', '_malloc', '_free', - 'addFunction', 'removeFunction' + 'addFunction', 'removeFunction', + 'intArrayFromString' ].forEach(function(m){ if(undefined === (api.wasm[m] = SQM[m])){ toss("Internal init error: Module."+m+" not found."); } }); + /** + The array object which holds the raw bytes managed by the + _malloc() binding. Side note: why on earth _malloc() manages + HEAP8 (an Int8Array), rather than HEAPU8 (a Uint8Array), is a + mystery. + */ + api.wasm._malloc.HEAP = api.wasm.HEAP8; /* What follows is colloquially known as "OO API #1". It is a binding of the sqlite3 API which is designed to be run within @@ -374,12 +426,6 @@ Module.postRun.push(function(namespace/*the module object, the target for the sqlite3 binding if, e.g., the wrapper is in the main thread and the sqlite3 API is in a worker. */ - /** Throws a new error, concatenating all args with a space between - each. */ - const toss = function(){ - throw new Error(Array.prototype.join.call(arguments, ' ')); - }; - /** The DB class wraps a sqlite3 db handle. @@ -954,10 +1000,9 @@ Module.postRun.push(function(namespace/*the module object, the target for if(null===val) { api.sqlite3_result_null(pCx); break; - }else if(undefined!==val.length){ - const pBlob = - api.wasm.allocate(val); - api.sqlite3_result_blob(pCx, pBlob, val.length, + }else if(isSupportedTypedArray(val)){ + const pBlob = api.wasm.mallocFromTypedArray(val); + api.sqlite3_result_blob(pCx, pBlob, val.byteLength, api.SQLITE_TRANSIENT); api.wasm._free(pBlob); break; @@ -1068,8 +1113,8 @@ Module.postRun.push(function(namespace/*the module object, the target for case BindTypes.string: return t; default: - if(v instanceof Uint8Array) return BindTypes.blob; - return undefined; + //console.log("isSupportedBindType",t,v); + return isSupportedTypedArray(v) ? BindTypes.blob : undefined; } }; @@ -1078,7 +1123,8 @@ Module.postRun.push(function(namespace/*the module object, the target for function returns that value, else it throws. */ const affirmSupportedBindType = function(v){ - return isSupportedBindType(v) || toss("Unsupport bind() argument type."); + //console.log('affirmSupportedBindType',v); + return isSupportedBindType(v) || toss("Unsupported bind() argument type:",typeof v); }; /** @@ -1140,11 +1186,12 @@ Module.postRun.push(function(namespace/*the module object, the target for if(!f._){ f._ = { string: function(stmt, ndx, val, asBlob){ - const bytes = intArrayFromString(val,true); - const pStr = api.wasm.allocate(bytes); + const bytes = api.wasm.intArrayFromString(val,true); + const pStr = api.wasm._malloc(bytes.length || 1); stmt._allocs.push(pStr); + api.wasm._malloc.HEAP.set(bytes, pStr); const func = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; - return func(stmt._pStmt, ndx, pStr, bytes.length, 0); + return func(stmt._pStmt, ndx, pStr, bytes.length, api.SQLITE_STATIC); } }; } @@ -1175,17 +1222,21 @@ Module.postRun.push(function(namespace/*the module object, the target for if('string'===typeof val){ rc = f._.string(stmt, ndx, val, true); }else{ - const len = val.length; - if(undefined===len){ + if(!isSupportedTypedArray(val)){ toss("Binding a value as a blob requires", - "that it have a length member."); + "that it be a string, Uint8Array, or Int8Array."); } - const pBlob = api.wasm.allocate(val); + //console.debug("Binding blob",len,val); + const pBlob = api.wasm.mallocFromTypedArray(val); stmt._allocs.push(pBlob); - rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, len, 0); + rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, val.byteLength, + api.SQLITE_STATIC); } + break; } - default: toss("Unsupported bind() argument type."); + default: + console.warn("Unsupported bind() argument type:",val); + toss("Unsupported bind() argument type."); } if(rc) stmt.db.checkRc(rc); return stmt; @@ -1286,7 +1337,9 @@ Module.postRun.push(function(namespace/*the module object, the target for - Strings are bound as strings (use bindAsBlob() to force blob binding). - - Uint8Array instances are bound as blobs. + - Uint8Array and Int8Array instances are bound as blobs. + (TODO: support binding other TypedArray types with larger + int sizes.) If passed an array, each element of the array is bound at the parameter index equal to the array index plus 1 @@ -1297,7 +1350,7 @@ Module.postRun.push(function(namespace/*the module object, the target for bindable parameter names, including any `$`, `@`, or `:` prefix. Because `$` is a legal identifier chararacter in JavaScript, that is the suggested prefix for bindable - parameters. + parameters: `stmt.bind({$a: 1, $b: 2})`. It returns this object on success and throws on error. Errors include: @@ -1342,8 +1395,9 @@ Module.postRun.push(function(namespace/*the module object, the target for arg.forEach((v,i)=>bindOne(this, i+1, affirmSupportedBindType(v), v)); return this; } - else if('object'===typeof arg/*null was checked above*/){ - /* bind by name */ + else if('object'===typeof arg/*null was checked above*/ + && !isSupportedTypedArray(arg)){ + /* Treat each property of arg as a named bound parameter. */ if(1!==arguments.length){ toss("When binding an object, an index argument is not permitted."); } @@ -1353,25 +1407,25 @@ Module.postRun.push(function(namespace/*the module object, the target for arg[k])); return this; }else{ - return bindOne(this, ndx, - affirmSupportedBindType(arg), arg); + return bindOne(this, ndx, affirmSupportedBindType(arg), arg); } toss("Should not reach this point."); }, /** - Special case of bind() which binds the given value - using the BLOB binding mechanism instead of the default - selected one for the value. The ndx may be a numbered - or named bind index. The value must be of type string, - Uint8Array, or null/undefined (both treated as null). + Special case of bind() which binds the given value using + the BLOB binding mechanism instead of the default selected + one for the value. The ndx may be a numbered or named bind + index. The value must be of type string, null/undefined + (both treated as null), or a TypedArray of a type supported + by the bind() API. If passed a single argument, a bind index of 1 is assumed. */ bindAsBlob: function(ndx,arg){ affirmStmtOpen(this); if(1===arguments.length){ + arg = ndx; ndx = 1; - arg = arguments[0]; } const t = affirmSupportedBindType(arg); if(BindTypes.string !== t && BindTypes.blob !== t diff --git a/ext/fiddle/testing1.js b/ext/fiddle/testing1.js index 0b178d8d9e..cc84ef6ef7 100644 --- a/ext/fiddle/testing1.js +++ b/ext/fiddle/testing1.js @@ -77,6 +77,12 @@ }); T.assert(2 === list.length); //log("Exec'd SQL:", list); + + let blob = db.selectValue("select b from t where a='blob'"); + T.assert(blob instanceof Uint8Array). + assert(0x68===blob[0] && 0x69===blob[1]); + blob = null; + let counter = 0, colNames = []; list.length = 0; db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ @@ -121,9 +127,7 @@ } }).createFunction({ name: "asis", - callback: function(arg){ - return arg; - } + callback: (arg)=>arg }); log("Testing DB::selectValue() w/ UDF..."); @@ -132,11 +136,7 @@ assert(3===db.selectValue("select bar(1,2)")). assert(-1===db.selectValue("select bar(1,2,-4)")). assert('hi'===db.selectValue("select asis('hi')")); - let blob = db.selectValue("select asis(X'6869')"); - T.assert(blob instanceof Uint8Array). - assert(2 === blob.length). - assert(0x68==blob[0] && 0x69==blob[1]); - + const eqApprox = function(v1,v2,factor=0.05){ //log('eqApprox',v1, v2); return v1>=(v2-factor) && v1<=(v2+factor); @@ -150,6 +150,27 @@ assert(eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). assert(eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))) ; + + log("Testing binding and UDF propagation of blobs..."); + let blobArg = new Uint8Array(2); + blobArg.set([0x68, 0x69], 0); + let blobRc = db.selectValue("select asis(?1)", blobArg); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length). + assert(0x68==blobRc[0] && 0x69==blobRc[1]); + blobRc = db.selectValue("select asis(X'6869')"); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length). + assert(0x68==blobRc[0] && 0x69==blobRc[1]); + + blobArg = new Int8Array(2); + blobArg.set([0x68, 0x69]); + console.debug("blobArg=",blobArg); + blobRc = db.selectValue("select asis(?1)", blobArg); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length); + console.debug("blobRc=",blobRc); + T.assert(0x68==blobRc[0] && 0x69==blobRc[1]); }; const testAttach = function(db){ diff --git a/manifest b/manifest index ab24710ad4..e5790b9c50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm:\scorrected\sthe\spropagation\sof\stext/blob\svalues\svia\sUDFs.\sDB.exec()'s\ssql\smay\snow\sbe\san\sarray\sof\sstrings\swhich\sget\sconcatenated\stogether\sbefore\spassing\sit\son\sto\ssqlite3_prepare_v2().\sDB.exec()'s\scallback\snow\sapplies\sto\sthe\sfirst\sstatement\swhich\shas\sresult\scolumns\sinstead\sof\sonly\sthe\sfirst\sstatement.\sFixed\sa\sprecedence\sbut\swhich\scaused\sisInt32()\sto\sreport\sfalse\spositives. -D 2022-06-25T03:53:43.503 +C wasm:\seliminated\sthe\sdependency\son\sthe\sdeprecated\semcc-provided\sallocate()\sfunction.\sAdjacent\scleanups\sin\sblob\sbinding. +D 2022-06-25T06:46:22.284 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -57,7 +57,7 @@ F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaed F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 F ext/fiddle/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api 540b9dec63a3a62a256e2f030827848a92e9b9d9b6fa5c0188295a4a1c5382cd -F ext/fiddle/EXPORTED_RUNTIME_METHODS b831017ba67ba993b34a27400cef2f6095bd6789c0fc4eba7e7a251c207be31c +F ext/fiddle/EXPORTED_RUNTIME_METHODS e499bbb5201bf671850da8dcd47de31b0db47c1183f0c669016801b4d2035534 F ext/fiddle/Makefile e25d34a0e1324f771d64c09c592601b97219282011587e6ce410fa8acdedb913 F ext/fiddle/SqliteTestUtil.js 559731c3e8e0de330ec7d292e6c1846566408caee6637acc8a119ac338a8781c F ext/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f @@ -65,11 +65,11 @@ F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba83 F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf -F ext/fiddle/sqlite3-api.js b706be1f777a4508f9e3e237301e11c108933ae651b3fa599bf047cac34b5bdb +F ext/fiddle/sqlite3-api.js 706bb55b3901a90ec3a0e588f00d4c260d61d5b5ac916b894f48cdd6203ff0ba F ext/fiddle/sqlite3-worker.js a9c2b614beca187dbdd8c053ec2770cc61ec1ac9c0ec6398ceb49a79f705a421 F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a -F ext/fiddle/testing1.js 3e5f1fb22764ec735396db518b85ea6b3070f6c2da479cba69c8d8e89f8299f8 +F ext/fiddle/testing1.js f3a5bd125154a51dfe20b699cb90c7cf2a23fbb5eaf9129cd4fb06e262de2647 F ext/fiddle/testing2.html 9063b2430ade2fe9da4e711addd1b51a2741cf0c7ebf6926472a5e5dd63c0bc4 F ext/fiddle/testing2.js 7b45b4e7fddbd51dbaf89b6722c02758051b34bac5a98c11b569a7e7572f88ee F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 42dc500819bfc1308a9542aa2cae4f6dfd98a29237c59cec82e0e6f9e0bf3779 -R a152bf70e7d16ac14f437dceeb13aaae +P 37a8fecb56793fbd3763a2240a0bd62b639c811934d3af2ef0e5ff579073d632 +R 4c023b8516cd5b1336a52ada20a79c7d U stephan -Z 385f37f8ee9972f16da6e6bace96df52 +Z 05e90a0a470a56e05a8109a17f4f47da # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cd66bdca9e..1bb4a79eb2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -37a8fecb56793fbd3763a2240a0bd62b639c811934d3af2ef0e5ff579073d632 \ No newline at end of file +140618b212e3aa9ff2df20f22af846a1c4c5ffaeefd84330446f61362b39a8f1 \ No newline at end of file From 1c67ec5692ff02b506aca335d60f387a4f3b75a1 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 25 Jun 2022 07:42:24 +0000 Subject: [PATCH 008/151] wasm: eliminated the need for Stmt objects to keep ahold of memory allocated for bound strings and blobs. Added alternate string/blob bind impls which are hypothetically more efficient but not yet proven to be so. FossilOrigin-Name: da1d3151a440567f34a2f6c0b2bfc2e9fab81c256cc361c9ce7b46f2c23a2aa8 --- ext/fiddle/EXPORTED_RUNTIME_METHODS | 1 + ext/fiddle/sqlite3-api.js | 75 +++++++++++++++++------------ ext/fiddle/testing1.js | 15 +++--- manifest | 16 +++--- manifest.uuid | 2 +- 5 files changed, 64 insertions(+), 45 deletions(-) diff --git a/ext/fiddle/EXPORTED_RUNTIME_METHODS b/ext/fiddle/EXPORTED_RUNTIME_METHODS index 3d06c32192..af2c48a3f4 100644 --- a/ext/fiddle/EXPORTED_RUNTIME_METHODS +++ b/ext/fiddle/EXPORTED_RUNTIME_METHODS @@ -11,3 +11,4 @@ setValue stackAlloc stackRestore stackSave +stringToUTF8Array diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js index eabac6cc5a..450bbb62ca 100644 --- a/ext/fiddle/sqlite3-api.js +++ b/ext/fiddle/sqlite3-api.js @@ -405,7 +405,7 @@ Module.postRun.push(function(namespace/*the module object, the target for ['getValue','setValue', 'stackSave', 'stackRestore', 'stackAlloc', 'allocateUTF8OnStack', '_malloc', '_free', 'addFunction', 'removeFunction', - 'intArrayFromString' + 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array' ].forEach(function(m){ if(undefined === (api.wasm[m] = SQM[m])){ toss("Internal init error: Module."+m+" not found."); @@ -524,7 +524,6 @@ Module.postRun.push(function(namespace/*the module object, the target for this._pStmt = arguments[1]; this.columnCount = api.sqlite3_column_count(this._pStmt); this.parameterCount = api.sqlite3_bind_parameter_count(this._pStmt); - this._allocs = [/*list of alloc'd memory blocks for bind() values*/] }; /** Throws if the given DB has been closed, else it is returned. */ @@ -1186,12 +1185,29 @@ Module.postRun.push(function(namespace/*the module object, the target for if(!f._){ f._ = { string: function(stmt, ndx, val, asBlob){ - const bytes = api.wasm.intArrayFromString(val,true); - const pStr = api.wasm._malloc(bytes.length || 1); - stmt._allocs.push(pStr); - api.wasm._malloc.HEAP.set(bytes, pStr); - const func = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; - return func(stmt._pStmt, ndx, pStr, bytes.length, api.SQLITE_STATIC); + if(1){ + /* _Hypothetically_ more efficient than the impl in the 'else' block. */ + const stack = api.wasm.stackSave(); + try{ + const n = api.wasm.lengthBytesUTF8(val)+1/*required for NUL terminator*/; + const pStr = api.wasm.stackAlloc(n); + api.wasm.stringToUTF8Array(val, api.wasm.HEAP8, pStr, n); + const f = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; + return f(stmt._pStmt, ndx, pStr, n-1, api.SQLITE_TRANSIENT); + }finally{ + api.wasm.stackRestore(stack); + } + }else{ + const bytes = api.wasm.intArrayFromString(val,true); + const pStr = api.wasm._malloc(bytes.length || 1); + api.wasm._malloc.HEAP.set(bytes.length ? bytes : [0], pStr); + try{ + const f = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; + return f(stmt._pStmt, ndx, pStr, bytes.length, api.SQLITE_TRANSIENT); + }finally{ + api.wasm._free(pStr); + } + } } }; } @@ -1221,16 +1237,28 @@ Module.postRun.push(function(namespace/*the module object, the target for case BindTypes.blob: { if('string'===typeof val){ rc = f._.string(stmt, ndx, val, true); - }else{ - if(!isSupportedTypedArray(val)){ - toss("Binding a value as a blob requires", - "that it be a string, Uint8Array, or Int8Array."); + }else if(!isSupportedTypedArray(val)){ + toss("Binding a value as a blob requires", + "that it be a string, Uint8Array, or Int8Array."); + }else if(1){ + /* _Hypothetically_ more efficient than the impl in the 'else' block. */ + const stack = api.wasm.stackSave(); + try{ + const pBlob = api.wasm.stackAlloc(val.byteLength || 1); + api.wasm.HEAP8.set(val.byteLength ? val : [0], pBlob) + rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, val.byteLength, + api.SQLITE_TRANSIENT); + }finally{ + api.wasm.stackRestore(stack); } - //console.debug("Binding blob",len,val); + }else{ const pBlob = api.wasm.mallocFromTypedArray(val); - stmt._allocs.push(pBlob); - rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, val.byteLength, - api.SQLITE_STATIC); + try{ + rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, val.byteLength, + api.SQLITE_TRANSIENT); + }finally{ + api.wasm._free(pBlob); + } } break; } @@ -1241,16 +1269,6 @@ Module.postRun.push(function(namespace/*the module object, the target for if(rc) stmt.db.checkRc(rc); return stmt; }; - - /** Frees any memory explicitly allocated for the given - Stmt object. Returns stmt. */ - const freeBindMemory = function(stmt){ - let m; - while(undefined !== (m = stmt._allocs.pop())){ - api.wasm._free(m); - } - return stmt; - }; Stmt.prototype = { /** @@ -1262,7 +1280,6 @@ Module.postRun.push(function(namespace/*the module object, the target for finalize: function(){ if(this._pStmt){ affirmUnlocked(this,'finalize()'); - freeBindMemory(this); delete this.db._statements[this._pStmt]; api.sqlite3_finalize(this._pStmt); delete this.columnCount; @@ -1275,9 +1292,7 @@ Module.postRun.push(function(namespace/*the module object, the target for /** Clears all bound values. Returns this object. Throws if this statement has been finalized. */ clearBindings: function(){ - freeBindMemory( - affirmUnlocked(affirmStmtOpen(this), 'clearBindings()') - ); + affirmUnlocked(affirmStmtOpen(this), 'clearBindings()') api.sqlite3_clear_bindings(this._pStmt); this._mayGet = false; return this; diff --git a/ext/fiddle/testing1.js b/ext/fiddle/testing1.js index cc84ef6ef7..9c17115f38 100644 --- a/ext/fiddle/testing1.js +++ b/ext/fiddle/testing1.js @@ -16,6 +16,7 @@ (function(){ const T = self.SqliteTestUtil; const log = console.log.bind(console); + const debug = console.debug.bind(console); const assert = function(condition, text) { if (!condition) { @@ -31,7 +32,7 @@ new TextEncoder('utf-8').encode("select 3 as a") /* Testing handling of Uint8Array input */ ); - //log("statement =",st); + //debug("statement =",st); T.assert(st._pStmt) .assert(!st._mayGet) .assert('a' === st.getColumnName(0)) @@ -76,7 +77,7 @@ /* Achtung: ^^^ bind support might be removed from multi-mode exec. */ }); T.assert(2 === list.length); - //log("Exec'd SQL:", list); + //debug("Exec'd SQL:", list); let blob = db.selectValue("select b from t where a='blob'"); T.assert(blob instanceof Uint8Array). @@ -138,7 +139,7 @@ assert('hi'===db.selectValue("select asis('hi')")); const eqApprox = function(v1,v2,factor=0.05){ - //log('eqApprox',v1, v2); + //debug('eqApprox',v1, v2); return v1>=(v2-factor) && v1<=(v2+factor); }; @@ -165,11 +166,11 @@ blobArg = new Int8Array(2); blobArg.set([0x68, 0x69]); - console.debug("blobArg=",blobArg); + //debug("blobArg=",blobArg); blobRc = db.selectValue("select asis(?1)", blobArg); T.assert(blobRc instanceof Uint8Array). assert(2 === blobRc.length); - console.debug("blobRc=",blobRc); + //debug("blobRc=",blobRc); T.assert(0x68==blobRc[0] && 0x69==blobRc[1]); }; @@ -196,9 +197,10 @@ const sqlite3 = Module.sqlite3; const api = sqlite3.api; const oo = sqlite3.SQLite3; - console.log("Loaded module:",api.sqlite3_libversion(), + log("Loaded module:",api.sqlite3_libversion(), api.sqlite3_sourceid()); log("Build options:",oo.compileOptionUsed()); + log("api.wasm.HEAP8 size =",api.wasm.HEAP8.length); const db = new oo.DB(); try { log("DB:",db.filename); @@ -213,6 +215,7 @@ db.close(); } log("Total Test count:",T.counter); + log("api.wasm.HEAP8 size =",api.wasm.HEAP8.length); }; initSqlite3Module(self.sqlite3TestModule).then(function(theModule){ diff --git a/manifest b/manifest index e5790b9c50..58086c0033 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm:\seliminated\sthe\sdependency\son\sthe\sdeprecated\semcc-provided\sallocate()\sfunction.\sAdjacent\scleanups\sin\sblob\sbinding. -D 2022-06-25T06:46:22.284 +C wasm:\seliminated\sthe\sneed\sfor\sStmt\sobjects\sto\skeep\sahold\sof\smemory\sallocated\sfor\sbound\sstrings\sand\sblobs.\sAdded\salternate\sstring/blob\sbind\simpls\swhich\sare\shypothetically\smore\sefficient\sbut\snot\syet\sproven\sto\sbe\sso. +D 2022-06-25T07:42:24.832 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -57,7 +57,7 @@ F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaed F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 F ext/fiddle/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api 540b9dec63a3a62a256e2f030827848a92e9b9d9b6fa5c0188295a4a1c5382cd -F ext/fiddle/EXPORTED_RUNTIME_METHODS e499bbb5201bf671850da8dcd47de31b0db47c1183f0c669016801b4d2035534 +F ext/fiddle/EXPORTED_RUNTIME_METHODS a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 F ext/fiddle/Makefile e25d34a0e1324f771d64c09c592601b97219282011587e6ce410fa8acdedb913 F ext/fiddle/SqliteTestUtil.js 559731c3e8e0de330ec7d292e6c1846566408caee6637acc8a119ac338a8781c F ext/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f @@ -65,11 +65,11 @@ F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba83 F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf -F ext/fiddle/sqlite3-api.js 706bb55b3901a90ec3a0e588f00d4c260d61d5b5ac916b894f48cdd6203ff0ba +F ext/fiddle/sqlite3-api.js 664a454d82694c6cfab36b8ad9e36620cfc44fa37e05642ba87109c37409f51f F ext/fiddle/sqlite3-worker.js a9c2b614beca187dbdd8c053ec2770cc61ec1ac9c0ec6398ceb49a79f705a421 F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a -F ext/fiddle/testing1.js f3a5bd125154a51dfe20b699cb90c7cf2a23fbb5eaf9129cd4fb06e262de2647 +F ext/fiddle/testing1.js 5c04721d205b3d909ec6896682743051cb5e5202e9151a058e90600a42f48f48 F ext/fiddle/testing2.html 9063b2430ade2fe9da4e711addd1b51a2741cf0c7ebf6926472a5e5dd63c0bc4 F ext/fiddle/testing2.js 7b45b4e7fddbd51dbaf89b6722c02758051b34bac5a98c11b569a7e7572f88ee F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 37a8fecb56793fbd3763a2240a0bd62b639c811934d3af2ef0e5ff579073d632 -R 4c023b8516cd5b1336a52ada20a79c7d +P 140618b212e3aa9ff2df20f22af846a1c4c5ffaeefd84330446f61362b39a8f1 +R ecfc68999ba200e5f9d0ce8c6a78fe70 U stephan -Z 05e90a0a470a56e05a8109a17f4f47da +Z ec8563cad502dab169a1c4538eaf6ec1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1bb4a79eb2..63c94d8ff4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -140618b212e3aa9ff2df20f22af846a1c4c5ffaeefd84330446f61362b39a8f1 \ No newline at end of file +da1d3151a440567f34a2f6c0b2bfc2e9fab81c256cc361c9ce7b46f2c23a2aa8 \ No newline at end of file From fa17e10592371a28182edbabae3a7a9b748fcca5 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 25 Jun 2022 10:30:24 +0000 Subject: [PATCH 009/151] wasm: added utility C code to generate a JSON-format "enum" of the numerous SQLITE_xyz constants so that we do not risk those getting out of sync in the JS code. Renamed initSqlite3Module to sqlite3InitModule. Cleanups in the TypedArray handling. FossilOrigin-Name: 778062e3b415dca5104eee398950741b6dbb9d4bdf7c998eef18371a42669946 --- Makefile.in | 6 +- ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api | 1 + ext/fiddle/SqliteTestUtil.js | 2 +- ext/fiddle/sqlite3-api.js | 89 ++++++------------ ext/fiddle/sqlite3-worker.js | 4 +- ext/fiddle/testing1.js | 17 ++-- ext/fiddle/wasm_util.c | 108 ++++++++++++++++++++++ manifest | 23 ++--- manifest.uuid | 2 +- 9 files changed, 168 insertions(+), 84 deletions(-) create mode 100644 ext/fiddle/wasm_util.c diff --git a/Makefile.in b/Makefile.in index da75b76a59..2e0768b829 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1543,16 +1543,16 @@ $(fiddle_module_js): Makefile sqlite3.c shell.c \ sqlite3.c shell.c gzip < $@ > $@.gz gzip < $(fiddle_dir)/fiddle-module.wasm > $(fiddle_dir)/fiddle-module.wasm.gz -$(sqlite3_wasm_js): Makefile sqlite3.c \ +$(sqlite3_wasm_js): Makefile sqlite3.c $(fiddle_dir)/wasm_util.c \ $(fiddle_dir)/sqlite3-api.js \ $(fiddle_dir)/EXPORTED_RUNTIME_METHODS \ $(fiddle_dir)/EXPORTED_FUNCTIONS.sqlite3-api emcc -o $@ $(emcc_flags) \ - -sEXPORT_NAME=initSqlite3Module \ + -sEXPORT_NAME=sqlite3InitModule \ -sEXPORTED_FUNCTIONS=@$(fiddle_dir_abs)/EXPORTED_FUNCTIONS.sqlite3-api \ --post-js=$(fiddle_dir)/sqlite3-api.js \ --no-entry \ - sqlite3.c + sqlite3.c $(fiddle_dir)/wasm_util.c gzip < $@ > $@.gz gzip < $(sqlite3_wasm) > $(sqlite3_wasm).gz gzip < $(fiddle_dir)/sqlite3-api.js > $(fiddle_dir)/sqlite3-api.js.gz diff --git a/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api b/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api index 3127b294d9..59dfef45b6 100644 --- a/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api @@ -48,4 +48,5 @@ _sqlite3_value_bytes _sqlite3_value_double _sqlite3_value_text _sqlite3_value_type +_sqlite3_wasm_enum_json _free diff --git a/ext/fiddle/SqliteTestUtil.js b/ext/fiddle/SqliteTestUtil.js index cf78946dc7..bcbdc59c51 100644 --- a/ext/fiddle/SqliteTestUtil.js +++ b/ext/fiddle/SqliteTestUtil.js @@ -90,7 +90,7 @@ /** This is a module object for use with the emscripten-installed - initSqlite3Module() factory function. + sqlite3InitModule() factory function. */ self.sqlite3TestModule = { postRun: [ diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js index 450bbb62ca..d4a5993ba5 100644 --- a/ext/fiddle/sqlite3-api.js +++ b/ext/fiddle/sqlite3-api.js @@ -13,11 +13,11 @@ This file is intended to be appended to the emcc-generated sqlite3.js via emcc: - emcc ... -sMODULARIZE -sEXPORT_NAME=initSqlite3Module --post-js=THIS_FILE + emcc ... -sMODULARIZE -sEXPORT_NAME=sqlite3InitModule --post-js=THIS_FILE It is loaded by importing the emcc-generated sqlite3.js, then: - initSqlite3Module({module object}).then( + sqlite3InitModule({module object}).then( function(theModule){ theModule.sqlite3 == an object containing this file's deliverables: @@ -29,7 +29,7 @@ It is up to the caller to provide a module object compatible with emcc, but it can be a plain empty object. The object passed to - initSqlite3Module() will get populated by the emscripten-generated + sqlite3InitModule() will get populated by the emscripten-generated bits and, in part, by the code from this file. Specifically, this file installs the `theModule.sqlite3` part shown above. @@ -151,49 +151,6 @@ Module.postRun.push(function(namespace/*the module object, the target for follows is strongly influenced by the sql.js implementation. */ const api = { - /* It is important that the following integer values match - those from the C code. Ideally we could fetch them from the - C API, e.g., in the form of a JSON object, but getting that - JSON string constructed within our current confines is - currently not worth the effort. - - Reminder to self: we could probably do so by adding the - proverbial level of indirection, calling in to C to get it, - and having that C func call an - emscripten-installed/JS-implemented library function which - builds the result object: - - const obj = {}; - sqlite3__get_enum(function(key,val){ - obj[key] = val; - }); - - but whether or not we can pass a function that way, via a - (void*) is as yet unknown. - */ - /* Minimum subset of sqlite result codes we'll need. */ - SQLITE_OK: 0, - SQLITE_ROW: 100, - SQLITE_DONE: 101, - /* sqlite data types */ - SQLITE_INTEGER: 1, - SQLITE_FLOAT: 2, - SQLITE_TEXT: 3, - SQLITE_BLOB: 4, - SQLITE_NULL: 5, - /* create_function() flags */ - SQLITE_DETERMINISTIC: 0x000000800, - SQLITE_DIRECTONLY: 0x000080000, - SQLITE_INNOCUOUS: 0x000200000, - /* sqlite encodings, used for creating UDFs, noting that we - will only support UTF8. */ - SQLITE_UTF8: 1, - /* Values for the final argument of sqlite3_result_blob(), - noting that these are interpreted in WASM as pointer - values. */ - SQLITE_TRANSIENT: -1, - SQLITE_STATIC: 0, - /** Holds state which are specific to the WASM-related infrastructure and glue code. It is not expected that client @@ -343,7 +300,18 @@ Module.postRun.push(function(namespace/*the module object, the target for "number", "number"]), }; - const uint8ToString = (str)=>new TextDecoder('utf-8').decode(str); + /* Import C-level constants... */ + //console.log("wasmEnum=",SQM.ccall('sqlite3_wasm_enum_json', 'string', [])); + const wasmEnum = JSON.parse(SQM.ccall('sqlite3_wasm_enum_json', 'string', [])); + ['resultCodes','dataTypes','udfFlags', + 'encodings','blobFinalizers'].forEach(function(t){ + Object.keys(wasmEnum[t]).forEach(function(k){ + api[k] = wasmEnum[t][k]; + }); + }); + + const utf8Decoder = new TextDecoder('utf-8'); + const typedArrayToString = (str)=>utf8Decoder.decode(str); //const stringToUint8 = (sql)=>new TextEncoder('utf-8').encode(sql); /** @@ -393,7 +361,7 @@ Module.postRun.push(function(namespace/*the module object, the target for parameters) requires using api.wasm.getValue(). */ api.sqlite3_prepare_v2 = function(pDb, sql, sqlLen, ppStmt, pzTail){ - if(sql instanceof Uint8Array) sql = uint8ToString(sql); + if(isSupportedTypedArray(sql)) sql = typedArrayToString(sql); switch(typeof sql){ case 'string': return prepareMethods.basic(pDb, sql, -1, ppStmt, null); case 'number': return prepareMethods.full(pDb, sql, -1, ppStmt, pzTail); @@ -556,7 +524,7 @@ Module.postRun.push(function(namespace/*the module object, the target for case 1: if('string'===typeof args[0]){ out.sql = args[0]; - }else if(args[0] instanceof Uint8Array){ + }else if(isSupportedTypedArray(args[0])){ out.sql = args[0]; }else if(args[0] && 'object'===typeof args[0]){ out.opt = args[0]; @@ -569,8 +537,8 @@ Module.postRun.push(function(namespace/*the module object, the target for break; default: toss("Invalid argument count for exec()."); }; - if(out.sql instanceof Uint8Array){ - out.sql = uint8ToString(out.sql); + if(isSupportedTypedArray(out.sql)){ + out.sql = typedArrayToString(out.sql); }else if(Array.isArray(out.sql)){ out.sql = out.sql.join(''); }else if('string'!==typeof out.sql){ @@ -829,8 +797,8 @@ Module.postRun.push(function(namespace/*the module object, the target for (opt.callback && opt.rowMode) ? opt.rowMode : false); try{ - const sql = (arg.sql instanceof Uint8Array) - ? uint8ToString(arg.sql) + const sql = isSupportedTypedArray(arg.sql) + ? typedArrayToString(arg.sql) : arg.sql; let pSql = api.wasm.allocateUTF8OnStack(sql) const ppStmt = api.wasm.stackAlloc(8) /* output (sqlite3_stmt**) arg */; @@ -1638,7 +1606,9 @@ Module.postRun.push(function(namespace/*the module object, the target for If passed no arguments then it returns an object mapping all known compilation options to their compile-time values, - or boolean true if they are defined with no value. + or boolean true if they are defined with no value. This + result, which is relatively expensive to compute, is cached + and returned for future no-argument calls. In all other cases it returns true if the given option was active when when compiling the sqlite3 module, else false. @@ -1649,7 +1619,8 @@ Module.postRun.push(function(namespace/*the module object, the target for */ compileOptionUsed: function f(optName){ if(!arguments.length){ - if(!f._opt){ + if(f._result) return f._result; + else if(!f._opt){ f._rx = /^([^=]+)=(.+)/; f._rxInt = /^-?\d+$/; f._opt = function(opt, rv){ @@ -1664,16 +1635,14 @@ Module.postRun.push(function(namespace/*the module object, the target for f._opt(k,ov); rc[ov[0]] = ov[1]; } - return rc; - } - else if(Array.isArray(optName)){ + return f._result = rc; + }else if(Array.isArray(optName)){ const rc = {}; optName.forEach((v)=>{ rc[v] = api.sqlite3_compileoption_used(v); }); return rc; - } - else if('object' === typeof optName){ + }else if('object' === typeof optName){ Object.keys(optName).forEach((k)=> { optName[k] = api.sqlite3_compileoption_used(k); }); diff --git a/ext/fiddle/sqlite3-worker.js b/ext/fiddle/sqlite3-worker.js index fe7423f4c4..ed369ad35b 100644 --- a/ext/fiddle/sqlite3-worker.js +++ b/ext/fiddle/sqlite3-worker.js @@ -20,7 +20,7 @@ loading sqlite3.js via a Worker. Loading sqlite3.js from the main window thread elides the Worker-specific API. Instantiating a worker with new Worker("sqlite.js") will not (cannot) call - initSqlite3Module() to initialize the module due to a + sqlite3InitModule() to initialize the module due to a timing/order-of-operations conflict (and that symbol is not exported in a way that a Worker loading it that way can see it). Thus JS code wanting to load the sqlite3 Worker-specific API needs to pass @@ -39,6 +39,6 @@ */ "use strict"; importScripts('sqlite3.js'); -initSqlite3Module().then(function(){ +sqlite3InitModule().then(function(){ setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'ready'}), 0); }); diff --git a/ext/fiddle/testing1.js b/ext/fiddle/testing1.js index 9c17115f38..44e285f73a 100644 --- a/ext/fiddle/testing1.js +++ b/ext/fiddle/testing1.js @@ -201,6 +201,13 @@ api.sqlite3_sourceid()); log("Build options:",oo.compileOptionUsed()); log("api.wasm.HEAP8 size =",api.wasm.HEAP8.length); + log("wasmEnum",JSON.parse(Module.ccall('sqlite3_wasm_enum_json', 'string', []))); + [ /* Spot-check a handful of constants to make sure they got installed... */ + 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', + 'SQLITE_STATIC', 'SQLITE_DIRECTONLY' + ].forEach(function(k){ + T.assert('number' === typeof api[k]); + }); const db = new oo.DB(); try { log("DB:",db.filename); @@ -215,14 +222,12 @@ db.close(); } log("Total Test count:",T.counter); - log("api.wasm.HEAP8 size =",api.wasm.HEAP8.length); }; - initSqlite3Module(self.sqlite3TestModule).then(function(theModule){ - /** Use a timeout so that we are (hopefully) out from - under the module init stack when our setup gets - run. Just on principle, not because we _need_ to - be. */ + sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){ + /** Use a timeout so that we are (hopefully) out from under + the module init stack when our setup gets run. Just on + principle, not because we _need_ to be. */ //console.debug("theModule =",theModule); setTimeout(()=>runTests(theModule), 0); }); diff --git a/ext/fiddle/wasm_util.c b/ext/fiddle/wasm_util.c new file mode 100644 index 0000000000..54326b2977 --- /dev/null +++ b/ext/fiddle/wasm_util.c @@ -0,0 +1,108 @@ +#include "sqlite3.h" +#include /*atexit()*/ +/* +** 2022-06-25 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +*********************************************************************** +** +** Utility functions for use with the emscripten/WASM bits. These +** functions ARE NOT part of the sqlite3 public API. They are strictly +** for internal use by the JS/WASM bindings. +*/ + +/** Result value of sqlite3_wasm_enum_json(). */ +static char * zWasmEnum = 0; +/* atexit() handler to clean up any WASM-related state. */ +static void sqlite3_wasm_cleanup(void){ + free(zWasmEnum); +} + +/* +** Returns a string containing a JSON-format "enum" of C-level +** constants intended to be imported into the JS environment. The JSON +** is initialized the first time this function is called and that +** result is reused for all future calls and cleaned up via atexit(). +** (If we didn't cache the result, it would be leaked by the JS glue +** code on each call during the WASM-to-JS conversion.) +** +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the JS/WASM bindings. +*/ +const char * sqlite3_wasm_enum_json(void){ + sqlite3_str * s; + if(zWasmEnum) return zWasmEnum; + s = sqlite3_str_new(0); + sqlite3_str_appendall(s, "{"); + +#define SD_(X,S,FINAL) \ + sqlite3_str_appendf(s, "\"%s\": %d%s", S, (int)X, (FINAL ? "}" : ", ")) +#define SD(X) SD_(X,#X,0) +#define SDFinal(X) SD_(X,#X,1) + + sqlite3_str_appendall(s,"\"resultCodes\": {"); + SD(SQLITE_OK); + SD(SQLITE_ERROR); + SD(SQLITE_INTERNAL); + SD(SQLITE_PERM); + SD(SQLITE_ABORT); + SD(SQLITE_BUSY); + SD(SQLITE_LOCKED); + SD(SQLITE_NOMEM); + SD(SQLITE_READONLY); + SD(SQLITE_INTERRUPT); + SD(SQLITE_IOERR); + SD(SQLITE_CORRUPT); + SD(SQLITE_NOTFOUND); + SD(SQLITE_FULL); + SD(SQLITE_CANTOPEN); + SD(SQLITE_PROTOCOL); + SD(SQLITE_EMPTY); + SD(SQLITE_SCHEMA); + SD(SQLITE_TOOBIG); + SD(SQLITE_CONSTRAINT); + SD(SQLITE_MISMATCH); + SD(SQLITE_MISUSE); + SD(SQLITE_NOLFS); + SD(SQLITE_AUTH); + SD(SQLITE_FORMAT); + SD(SQLITE_RANGE); + SD(SQLITE_NOTADB); + SD(SQLITE_NOTICE); + SD(SQLITE_WARNING); + SD(SQLITE_ROW); + SDFinal(SQLITE_DONE); + + sqlite3_str_appendall(s,",\"dataTypes\": {"); + SD(SQLITE_INTEGER); + SD(SQLITE_FLOAT); + SD(SQLITE_TEXT); + SD(SQLITE_BLOB); + SDFinal(SQLITE_NULL); + + sqlite3_str_appendf(s,",\"encodings\": {"); + SDFinal(SQLITE_UTF8); + + sqlite3_str_appendall(s,",\"blobFinalizers\": {"); + SD(SQLITE_STATIC); + SDFinal(SQLITE_TRANSIENT); + + sqlite3_str_appendall(s,",\"udfFlags\": {"); + SD(SQLITE_DETERMINISTIC); + SD(SQLITE_DIRECTONLY); + SDFinal(SQLITE_INNOCUOUS); + +#undef SD_ +#undef SD +#undef SDFinal + sqlite3_str_appendall(s, "}"); + zWasmEnum = sqlite3_str_finish(s); + atexit(sqlite3_wasm_cleanup); + return zWasmEnum; +} diff --git a/manifest b/manifest index 58086c0033..3697a203f1 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C wasm:\seliminated\sthe\sneed\sfor\sStmt\sobjects\sto\skeep\sahold\sof\smemory\sallocated\sfor\sbound\sstrings\sand\sblobs.\sAdded\salternate\sstring/blob\sbind\simpls\swhich\sare\shypothetically\smore\sefficient\sbut\snot\syet\sproven\sto\sbe\sso. -D 2022-06-25T07:42:24.832 +C wasm:\sadded\sutility\sC\scode\sto\sgenerate\sa\sJSON-format\s"enum"\sof\sthe\snumerous\sSQLITE_xyz\sconstants\sso\sthat\swe\sdo\snot\srisk\sthose\sgetting\sout\sof\ssync\sin\sthe\sJS\scode.\sRenamed\sinitSqlite3Module\sto\ssqlite3InitModule.\sCleanups\sin\sthe\sTypedArray\shandling. +D 2022-06-25T10:30:24.409 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in bccb0ed3f05fc41aee15da77c844c48b5da419cbb9af35b8a147536c9ad1c822 +F Makefile.in b3ccd1a79e6364d49c465b38cec5eccdc74dfdc06501be62ef8eb01dc1f93f43 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc de7cb3e095ce2fdc33513ccd76ebdaeda1483d0ddab0410fe65cbdeadd4c0ee1 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -56,22 +56,23 @@ F ext/expert/sqlite3expert.c 6ca30d73b9ed75bd56d6e0d7f2c962d2affaa72c505458619d0 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 F ext/fiddle/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 -F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api 540b9dec63a3a62a256e2f030827848a92e9b9d9b6fa5c0188295a4a1c5382cd +F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api 2ef8a8d58815a96de196d085b8aaf126400b8d13a5b7f724f86dfe9f61d0600d F ext/fiddle/EXPORTED_RUNTIME_METHODS a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 F ext/fiddle/Makefile e25d34a0e1324f771d64c09c592601b97219282011587e6ce410fa8acdedb913 -F ext/fiddle/SqliteTestUtil.js 559731c3e8e0de330ec7d292e6c1846566408caee6637acc8a119ac338a8781c +F ext/fiddle/SqliteTestUtil.js 2e87d424b12674476bdf8139934dcacc3ff8a7a5f5ff4392ba5e5a8d8cee9fbd F ext/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf -F ext/fiddle/sqlite3-api.js 664a454d82694c6cfab36b8ad9e36620cfc44fa37e05642ba87109c37409f51f -F ext/fiddle/sqlite3-worker.js a9c2b614beca187dbdd8c053ec2770cc61ec1ac9c0ec6398ceb49a79f705a421 +F ext/fiddle/sqlite3-api.js 03ac065f4bc68eefd3b09cf3957a58891cbfcebe2a5ffe1c8a10f16a58dfc70b +F ext/fiddle/sqlite3-worker.js 50b7a9ce14c8fae0af965e35605fe12cafb79c1e01e99216d8110d8b02fbf4b5 F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a -F ext/fiddle/testing1.js 5c04721d205b3d909ec6896682743051cb5e5202e9151a058e90600a42f48f48 +F ext/fiddle/testing1.js f9615ff58b9de6879e4836618b34322085510ec44c6754e725a92a097c908a6f F ext/fiddle/testing2.html 9063b2430ade2fe9da4e711addd1b51a2741cf0c7ebf6926472a5e5dd63c0bc4 F ext/fiddle/testing2.js 7b45b4e7fddbd51dbaf89b6722c02758051b34bac5a98c11b569a7e7572f88ee +F ext/fiddle/wasm_util.c b63e00c2f264ab4a9c45c9f9727627cbc4d8aa2f93c5dd09e8105d63ff7e0872 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea @@ -1978,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 140618b212e3aa9ff2df20f22af846a1c4c5ffaeefd84330446f61362b39a8f1 -R ecfc68999ba200e5f9d0ce8c6a78fe70 +P da1d3151a440567f34a2f6c0b2bfc2e9fab81c256cc361c9ce7b46f2c23a2aa8 +R ff53072845ce3be319c8a5687950f355 U stephan -Z ec8563cad502dab169a1c4538eaf6ec1 +Z 0ce9f393ac2f284b021257ae10338830 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 63c94d8ff4..7a1e3598fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da1d3151a440567f34a2f6c0b2bfc2e9fab81c256cc361c9ce7b46f2c23a2aa8 \ No newline at end of file +778062e3b415dca5104eee398950741b6dbb9d4bdf7c998eef18371a42669946 \ No newline at end of file From 0702c63bc0534ed0d8e49af45cfabfb482b54ea5 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 25 Jun 2022 18:18:45 +0000 Subject: [PATCH 010/151] wasm: lots of doc additions and refactoring. Refactored the WASM memory heap usage to hopefully eventually account for a runtime-growable heap. Differentiate between supported TypedArray types for input SQL strings vs binding/fetching blobs. Might (untested) have implemented the ability to bind UtfNNArray values as blobs, where NN is one of 16 or 32. FossilOrigin-Name: e10d57dfbaa672a3a4cbfd9a9209552c3bde15cc75af838690ca412fd182066a --- Makefile.in | 45 +++- ext/fiddle/sqlite3-api.js | 424 ++++++++++++++++++++++---------------- ext/fiddle/testing1.js | 13 +- ext/fiddle/wasm_util.c | 7 +- manifest | 18 +- manifest.uuid | 2 +- 6 files changed, 309 insertions(+), 200 deletions(-) diff --git a/Makefile.in b/Makefile.in index 2e0768b829..2d90cc0200 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1529,10 +1529,15 @@ sqlite3_wasm = $(fiddle_dir)/sqlite3.wasm #emcc_opt = -O2 #emcc_opt = -O3 emcc_opt = -Oz -emcc_flags = $(emcc_opt) -sALLOW_TABLE_GROWTH -sSTRICT_JS \ - -sENVIRONMENT=web -sMODULARIZE \ +emcc_flags = $(emcc_opt) \ + -sALLOW_TABLE_GROWTH \ + -sABORTING_MALLOC \ + -sSTRICT_JS \ + -sENVIRONMENT=web \ + -sMODULARIZE \ -sEXPORTED_RUNTIME_METHODS=@$(fiddle_dir_abs)/EXPORTED_RUNTIME_METHODS \ -sDYNAMIC_EXECUTION=0 \ + --minify 0 \ -I. $(SHELL_OPT) $(fiddle_module_js): Makefile sqlite3.c shell.c \ $(fiddle_dir)/EXPORTED_RUNTIME_METHODS \ @@ -1590,7 +1595,7 @@ wasm: fiddle sqlite3-wasm # # -sMODULARIZE: changes how the generated code is structured to avoid # declaring a global Module object and instead installing a function -# which loads and initialized the module. The function is named... +# which loads and initializes the module. The function is named... # # -sEXPORT_NAME=jsFunctionName (see -sMODULARIZE) # @@ -1610,7 +1615,15 @@ wasm: fiddle sqlite3-wasm # results in build errors. # # -sALLOW_TABLE_GROWTH is required for (at a minimum) the UDF-binding -# feature. +# feature. Without it, JS functions cannot be made to proxy C-side +# callbacks. +# +# -sABORTING_MALLOC causes the JS-bound _malloc() to abort rather than +# return 0 on OOM. If set to 0 then all code which uses _malloc() +# must, just like in C, check the result before using it, else +# they're likely to corrupt the JS/WASM heap by writing to its +# address of 0. It is, as of this writing, enabled in Emscripten by +# default but we enable it explicitly in case that default changes. # # -sDYNAMIC_EXECUTION=0 disables eval() and the Function constructor. # If the build runs without these, it's preferable to use this flag @@ -1642,4 +1655,28 @@ wasm: fiddle sqlite3-wasm # primarily because -Oz will shrink the wasm file notably. JS-side # minification makes little difference in terms of overall # distributable size. +# +# --minify 0: disables minification of the generated JS code, +# regardless of optimization level. Minification of the JS has +# minimal overall effect in the larger scheme of things and results +# in JS files which can neither be edited nor viewed as text files in +# Fossil (which flags them as binary because of their extreme line +# lengths). Interestingly, whether or not the comments in the +# generated JS file get stripped is unaffected by this setting and +# depends entirely on the optimization level. Higher optimization +# levels reduce the size of the JS considerably even without +# minification. +# +# ACHTUNG... +# +# -sALLOW_MEMORY_GROWTH is very tempting but has side-effects which +# may break the higher-level code. Specifically, growning the memory +# region beyong the default size requires that the app-visible arrays +# which act as views for that memory buffer have to be replaced, +# which inherently leaves any client-side references to those arrays +# with stale state, effectively resulting in memory corruption. +# Until/unless that can be well-encapsulated, such that the higher-level +# bindings which access those memory heaps are known to be using the +# proper references, DO NOT build with this option. +# ######################################################################## diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js index d4a5993ba5..5350464e90 100644 --- a/ext/fiddle/sqlite3-api.js +++ b/ext/fiddle/sqlite3-api.js @@ -40,47 +40,69 @@ Because using certain parts of the low-level API properly requires some degree of WASM-related magic, it is not recommended that that - API be used as-is in client-level code. Rather, client code should - use the higher-level OO API or write a custom wrapper on top of the - lower-level API. In short, most of the C-style API is used in an - intuitive manner from JS but any C-style APIs which take + API be used as-is in client-level code. Rather, client code is + encouraged use the higher-level OO API or write a custom wrapper on + top of the lower-level API. In short, most of the C-style API is + used in an intuitive manner from JS but any C-style APIs which take pointers-to-pointer arguments require WASM-specific interfaces - installed by emcscripten-generated code. Those which take or return + installed by Emscripten-generated code. Those which take or return only integers, doubles, strings, or "plain" pointers to db or statement objects can be used in "as normal," noting that "pointers" - in wasm are simply 32-bit integers. + in WASM are simply 32-bit integers. - # Goals and Non-goals of this API - Goals: + Specific goals of this project: - Except where noted in the non-goals, provide a more-or-less complete wrapper to the sqlite3 C API, insofar as WASM feature parity with C allows for. In fact, provide at least 3... - - (1) The aforementioned C-style API. (2) An OO-style API on - top of that, designed to run in the same thread (main window or - Web Worker) as the C API. (3) A less-capable wrapper which can - work across the main window/worker boundary, where the sqlite3 API - is one of those and this wrapper is in the other. That - constellation places some considerable limitations on how the API - can be interacted with, but keeping the DB operations out of the - UI thread is generally desirable. + 1) Bind a low-level sqlite3 API which is as close to the native + one as feasible in terms of usage. + + 2) A higher-level API, more akin to sql.js and node.js-style + implementations. This one speaks directly to the low-level + API. This API must be used from the same thread as the + low-level API. + + 3) A second higher-level API which speaks to the previous APIs via + worker messages. This one is intended for use in the main + thread, with the lower-level APIs installed in a Worker thread, + and talking to them via Worker messages. Because Workers are + asynchronouns and have only a single message channel, some + acrobatics are needed here to feed async work results back to + the client (as we cannot simply pass around callbacks between + the main and Worker threads). - Insofar as possible, support client-side storage using JS filesystem APIs. As of this writing, such things are still very much TODO. - Non-goals: + + Specific non-goals of this project: - As WASM is a web-centric technology and UTF-8 is the King of - Encodings in that realm, there are no current plans to support the - UTF16-related APIs. They would add a complication to the bindings - for no appreciable benefit. + Encodings in that realm, there are no currently plans to support + the UTF16-related sqlite3 APIs. They would add a complication to + the bindings for no appreciable benefit. - Supporting old or niche-market platforms. WASM is built for a modern web and requires modern platforms. + + Attribution: + + Though this code is not a direct copy/paste, much of the + functionality in this file is strongly influenced by the + corresponding features in sql.js: + + https://github.com/sql-js/sql.js + + sql.js was an essential stepping stone in this code's development as + it demonstrated how to handle some of the WASM-related voodoo (like + handling pointers-to-pointers and adding JS implementations of + C-bound callback functions). These APIs have a considerably + different shape than sql.js's, however. */ if(!Module.postRun) Module.postRun = []; /* ^^^^ the name Module is, in this setup, scope-local in the generated @@ -88,124 +110,208 @@ if(!Module.postRun) Module.postRun = []; Module.postRun.push(function(namespace/*the module object, the target for installed features*/){ 'use strict'; - /* For reference: sql.js does essentially everything we want and - it solves much of the wasm-related voodoo, but we'll need a - different structure because we want the db connection to run in - a worker thread and feed data back into the main - thread. Regardless of those differences, it makes a great point - of reference: - - https://github.com/sql-js/sql.js - - Some of the specific design goals here: - - - Bind a low-level sqlite3 API which is close to the native one - in terms of usage. - - - Create a higher-level one, more akin to sql.js and - node.js-style implementations. This one would speak directly - to the low-level API. This API could be used by clients who - import the low-level API directly into their main thread - (which we don't want to recommend but also don't want to - outright forbid). - - - Create a second higher-level one which speaks to the - low-level API via worker messages. This one would be intended - for use in the main thread, talking to the low-level UI via - worker messages. Because workers have only a single message - channel, some acrobatics will be needed here to feed async - work results back into client-side callbacks (as those - callbacks cannot simply be passed to the worker). Exactly - what those acrobatics should look like is not yet entirely - clear and much experimentation is pending. + /** */ - - const SQM = namespace/*the sqlite module object */; + const SQM/*interal-use convenience alias*/ = namespace/*the sqlite module object */; /** Throws a new Error, the message of which is the concatenation all args with a space between each. */ const toss = function(){ throw new Error(Array.prototype.join.call(arguments, ' ')); }; - - /** - Returns true if v appears to be one of our supported TypedArray types: - Uint8Array or Int8Array. - */ - const isSupportedTypedArray = function(v){ - return v && (undefined!==v.byteLength) && (v.byteLength === v.length); + + /** Returns true if n is a 32-bit (signed) integer, + else false. */ + const isInt32 = function(n){ + return (n===(n|0) && n<0xFFFFFFFF) ? true : undefined; }; - /** Returns true if isSupportedTypedArray(v) does, else throws with a message + /** Returns v if v appears to be a TypedArray, else false. */ + const isTypedArray = (v)=>{ + return (v && v.constructor && isInt32(v.constructor.BYTES_PER_ELEMENT)) ? v : false; + }; + + /** + Returns true if v appears to be one of our bind()-able + TypedArray types: Uint8Array or Int8Array. Support for + TypedArrays with element sizes >1 is a potential TODO. + */ + const isBindableTypedArray = (v)=>{ + return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + }; + + /** + Returns true if v appears to be one of the TypedArray types + which is legal for holding SQL code (as opposed to binary blobs). + + Currently this is the same as isBindableTypedArray() but it + seems likely that we'll eventually want to add Uint32Array + and friends to the isBindableTypedArray() list but not to the + isSQLableTypedArray() list. + */ + const isSQLableTypedArray = isBindableTypedArray; + + /** Returns true if isBindableTypedArray(v) does, else throws with a message that v is not a supported TypedArray value. */ - const affirmSupportedTypedArray = function(v){ - return isSupportedTypedArray(v) + const affirmBindableTypedArray = (v)=>{ + return isBindableTypedArray(v) || toss("Value is not of a supported TypedArray type."); }; /** - Set up the main sqlite3 binding API here, mimicking the C API as - closely as we can. - - Attribution: though not a direct copy/paste, much of what - follows is strongly influenced by the sql.js implementation. + The main sqlite3 binding API gets installed into this object, + mimicking the C API as closely as we can. The numerous members + names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as + possible, identically to the C-native counterparts. A very few + exceptions may require an additional level of proxy function, as + documented in this object. */ const api = { + /** + The sqlite3_prepare_v2() binding handles two different uses + with differing JS/WASM semantics: + + 1) sqlite3_prepare_v2(pDb, sqlString, -1, ppStmt [, null]) + + 2) sqlite3_prepare_v2(pDb, sqlPointer, -1, ppStmt, sqlPointerToPointer) + + Note that the SQL length argument (the 3rd argument) must + always be negative because it must be a byte length and + that value is expensive to calculate from JS (where only + the character length of strings is readily available). It + is retained in this API's interface for code/documentation + compatibility reasons but is currently _always_ + ignored. When using the 2nd form of this call, it is + critical that the custom-allocated string be terminated + with a 0 byte. (Potential TODO: if the 3rd argument is >0, + assume the caller knows precisely what they're doing, vis a + vis WASM memory management, and pass it on as-is. That + approach currently seems fraught with peril.) + + In usage (1), the 2nd argument must be of type string, + Uint8Array, or Int8Array (either of which is assumed to + hold SQL). If it is, this function assumes case (1) and + calls the underyling C function with: + + (pDb, sqlAsString, -1, ppStmt, null) + + The pzTail argument is ignored in this case because its result + is meaningless when a string-type value is passed through + (because the string goes through another level of internal + conversion for WASM's sake and the result pointer would refer + to that conversion's memory, not the passed-in string). + + If sql is not a string or supported TypedArray, it must be + a _pointer_ to a string which was allocated via + api.wasm.allocateUTF8OnStack(), api.wasm._malloc(), or + equivalent. In that case, + the final argument may be 0/null/undefined or must be a + pointer to which the "tail" of the compiled SQL is written, + as documented for the C-side sqlite3_prepare_v2(). In case + (2), the underlying C function is called with: + + (pDb, sqlAsPointer, -1, ppStmt, pzTail) + + It returns its result and compiled statement as documented + in the C API. Fetching the output pointers (4th and 5th + parameters) requires using api.wasm.getValue() and the + pzTail will point to an address relative to the + sqlAsPointer value. + + If passed an invalid 2nd argument type, this function will + throw. That behaviour is in strong constrast to all of the + other C-bound functions (which return a non-0 result code + on error) but is necessary because we have to way to set + the db's error state such that this function could return a + non-0 integer and the client could call sqlite3_errcode() + or sqlite3_errmsg() to fetch it. + */ + sqlite3_prepare_v2: undefined/*installed later*/, + /** Holds state which are specific to the WASM-related infrastructure and glue code. It is not expected that client code will normally need these, but they're exposed here in case it does. + + Note that a number of members of this object are injected + dynamically after the api object is fully constructed, so + not all are documented inline here. */ wasm: { /** api.wasm._malloc()'s srcTypedArray.byteLength bytes, - populates them with the values from the source array, - and returns the pointer to that memory. The pointer - must eventually be passed to api.wasm._free() to clean - it up. + populates them with the values from the source + TypedArray, and returns the pointer to that memory. The + returned pointer must eventually be passed to + api.wasm._free() to clean it up. As a special case, to avoid further special cases where this is used, if srcTypedArray.byteLength is 0, it - allocates a single byte and sets it to the value 0. + allocates a single byte and sets it to the value + 0. Even in such cases, calls must behave as if the + allocated memory has exactly srcTypedArray.byteLength + bytes. ACHTUNG: this currently only works for Uint8Array and - Int8Array types. + Int8Array types and will throw if srcTypedArray is of + any other type. */ mallocFromTypedArray: function(srcTypedArray){ - affirmSupportedTypedArray(srcTypedArray); + affirmBindableTypedArray(srcTypedArray); const pRet = api.wasm._malloc(srcTypedArray.byteLength || 1); - if(srcTypedArray.byteLength){ - api.wasm._malloc.HEAP.set(srcTypedArray, pRet); - /* That unfortunately does not behave intuitively - when copying, e.g., the contents of a - Uint16Array, copying only 1 byte of each - entry instead of blitting the whole array - contents over the destination array. A potential TODO - is handle that copying here so that we can support a wider - array (haha) of bindable-as-blob types. */ - } - else api.wasm._malloc.HEAP[pRet] = 0; + this.heapForSize(srcTypedArray).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); return pRet; }, + /** Convenience form of this.heapForSize(8,false). */ + HEAP8: ()=>SQM['HEAP8'], + /** Convenience form of this.heapForSize(8,true). */ + HEAPU8: ()=>SQM['HEAPU8'], /** - The TypedArray buffer which holds the heap memory - managed by the emscripten-installed _malloc(). + Requires n to be one of (8, 16, 32) or a TypedArray + instance of Int8Array, Int16Array, Int32Array, or their + Uint counterparts. + + Returns the current integer-based TypedArray view of + the WASM heap memory buffer associated with the given + block size. If unsigned is truthy then the "U" + (unsigned) variant of that view is returned, else the + signed variant is returned. If passed a TypedArray + value and no 2nd argument then the 2nd argument + defaults to the signedness of that array. Note that + Float32Array and Float64Array views are not supported + by this function. + + Note that growth of the heap will invalidate any + references to this heap, so do not hold a reference + longer than needed and do not use a reference + after any operation which may allocate. + + Throws if passed an invalid n */ - HEAP8: SQM.HEAP8 + heapForSize: function(n,unsigned = true){ + if(isTypedArray(n)){ + if(1===arguments.length){ + unsigned = n instanceof Uint8Array || n instanceof Uint16Array + || n instanceof Uint32Array; + } + n = n.constructor.BYTES_PER_ELEMENT * 8; + } + switch(n){ + case 8: return SQM[unsigned ? 'HEAPU8' : 'HEAP8']; + case 16: return SQM[unsigned ? 'HEAPU16' : 'HEAP16']; + case 32: return SQM[unsigned ? 'HEAPU32' : 'HEAP32']; + } + toss("Invalid heapForSize() size: expecting 8, 16, or 32."); + } } }; - [/* C-side functions to bind. Each entry is an array with 3 or 4 - elements: + [/* C-side functions to bind. Each entry is an array with 3 elements: ["c-side name", "result type" (cwrap() syntax), [arg types in cwrap() syntax] ] - - If it has 4 elements, the first one is an alternate name to - use for the JS-side binding. That's required when overloading - a binding for two different uses. */ ["sqlite3_bind_blob","number",["number", "number", "number", "number", "number"]], ["sqlite3_bind_double","number",["number", "number", "number"]], @@ -263,10 +369,7 @@ Module.postRun.push(function(namespace/*the module object, the target for ["sqlite3_value_text", "string", ["number"]], ["sqlite3_value_type", "number", ["number"]] //["sqlite3_normalized_sql", "string", ["number"]] - ].forEach(function(a){ - const k = (4==a.length) ? a.shift() : a[0]; - api[k] = SQM.cwrap.apply(this, a); - }); + ].forEach((a)=>api[a[0]] = SQM.cwrap.apply(this, a)); /** Proxies for variants of sqlite3_prepare_v2() which have @@ -314,54 +417,9 @@ Module.postRun.push(function(namespace/*the module object, the target for const typedArrayToString = (str)=>utf8Decoder.decode(str); //const stringToUint8 = (sql)=>new TextEncoder('utf-8').encode(sql); - /** - sqlite3_prepare_v2() binding which handles two different uses - with differing JS/WASM semantics: - - 1) sqlite3_prepare_v2(pDb, sqlString, -1, ppStmt [, null]) - - 2) sqlite3_prepare_v2(pDb, sqlPointer, -1, ppStmt, sqlPointerToPointer) - - Note that the SQL length argument (the 3rd argument) must - always be negative because it must be a byte length and that - value is expensive to calculate from JS (where we get the - character length of strings). It is retained in this API's - interface for code/documentation compatibility reasons but is - currently _always_ ignored. When using the 2nd form of this - call, it is critical that the custom-allocated string be - terminated with a 0 byte. (Potential TODO: if this value is >0, - assume the caller knows precisely what they're doing and pass - it on as-is. That approach currently seems fraught with peril.) - - In usage (1), the 2nd argument must be of type string or - Uint8Array (which is assumed to hold SQL). If it is, this - function assumes case (1) and calls the underling C function - with: - - (pDb, sqlAsString, -1, ppStmt, null) - - The pzTail argument is ignored in this case because its result - is meaningless when a string-type value is passed through - (because the string goes through another level of internal - conversion for WASM's sake and the result pointer would refer - to that conversion's memory, not the passed-in string). - - If sql is not a string or Uint8Array, it must be a _pointer_ to - a string which was allocated via api.wasm.allocateUTF8OnStack() - or equivalent (TODO: define "or equivalent"). In that case, the - final argument may be 0/null/undefined or must be a pointer to - which the "tail" of the compiled SQL is written, as documented - for the C-side sqlite3_prepare_v2(). In case (2), the - underlying C function is called with: - - (pDb, sqlAsPointer, -1, ppStmt, pzTail) - - It returns its result and compiled statement as documented in - the C API. Fetching the output pointers (4th and 5th - parameters) requires using api.wasm.getValue(). - */ + /* Documented inline in the api object. */ api.sqlite3_prepare_v2 = function(pDb, sql, sqlLen, ppStmt, pzTail){ - if(isSupportedTypedArray(sql)) sql = typedArrayToString(sql); + if(isSQLableTypedArray(sql)) sql = typedArrayToString(sql); switch(typeof sql){ case 'string': return prepareMethods.basic(pDb, sql, -1, ppStmt, null); case 'number': return prepareMethods.full(pDb, sql, -1, ppStmt, pzTail); @@ -369,23 +427,35 @@ Module.postRun.push(function(namespace/*the module object, the target for } }; - /** Populate api.wasm with several members of the module object... */ - ['getValue','setValue', 'stackSave', 'stackRestore', 'stackAlloc', - 'allocateUTF8OnStack', '_malloc', '_free', - 'addFunction', 'removeFunction', - 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array' + /** + Populate api.wasm with several members of the module object. Some of these + will be required by higher-level code. At a minimum: + + getValue(), setValue(), stackSave(), stackRestore(), stackAlloc(), _malloc(), + _free(), addFunction(), removeFunction() + + The rest are exposed primarily for internal use in this API but may well + be useful from higher-level client code. + + All of the functions injected here are part of the + Emscripten-exposed APIs and are documented "elsewhere". Some + are documented in the Emscripten-generated `sqlite3.js` and + some are documented (if at all) in places unknown, possibly + even inaccessible, to us. + */ + [ + // Memory management: + 'getValue','setValue', 'stackSave', 'stackRestore', 'stackAlloc', + 'allocateUTF8OnStack', '_malloc', '_free', + // String utilities: + 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', + // The obligatory "misc" category: + 'addFunction', 'removeFunction' ].forEach(function(m){ if(undefined === (api.wasm[m] = SQM[m])){ toss("Internal init error: Module."+m+" not found."); } }); - /** - The array object which holds the raw bytes managed by the - _malloc() binding. Side note: why on earth _malloc() manages - HEAP8 (an Int8Array), rather than HEAPU8 (a Uint8Array), is a - mystery. - */ - api.wasm._malloc.HEAP = api.wasm.HEAP8; /* What follows is colloquially known as "OO API #1". It is a binding of the sqlite3 API which is designed to be run within @@ -500,12 +570,6 @@ Module.postRun.push(function(namespace/*the module object, the target for return db; }; - /** Returns true if n is a 32-bit (signed) integer, - else false. */ - const isInt32 = function(n){ - return (n===(n|0) && n<0xFFFFFFFF) ? true : undefined; - }; - /** Expects to be passed (arguments) from DB.exec() and DB.execMulti(). Does the argument processing/validation, throws @@ -522,9 +586,7 @@ Module.postRun.push(function(namespace/*the module object, the target for const out = {opt:{}}; switch(args.length){ case 1: - if('string'===typeof args[0]){ - out.sql = args[0]; - }else if(isSupportedTypedArray(args[0])){ + if('string'===typeof args[0] || isSQLableTypedArray(args[0])){ out.sql = args[0]; }else if(args[0] && 'object'===typeof args[0]){ out.opt = args[0]; @@ -537,7 +599,7 @@ Module.postRun.push(function(namespace/*the module object, the target for break; default: toss("Invalid argument count for exec()."); }; - if(isSupportedTypedArray(out.sql)){ + if(isSQLableTypedArray(out.sql)){ out.sql = typedArrayToString(out.sql); }else if(Array.isArray(out.sql)){ out.sql = out.sql.join(''); @@ -797,7 +859,7 @@ Module.postRun.push(function(namespace/*the module object, the target for (opt.callback && opt.rowMode) ? opt.rowMode : false); try{ - const sql = isSupportedTypedArray(arg.sql) + const sql = isSQLableTypedArray(arg.sql) ? typedArrayToString(arg.sql) : arg.sql; let pSql = api.wasm.allocateUTF8OnStack(sql) @@ -939,7 +1001,8 @@ Module.postRun.push(function(namespace/*the module object, the target for const pBlob = api.sqlite3_value_blob(pVal); arg = new Uint8Array(n); let i; - for(i = 0; i < n; ++i) arg[i] = api.wasm.HEAP8[pBlob+i]; + const heap = n ? api.wasm.HEAP8() : false; + for(i = 0; i < n; ++i) arg[i] = heap[pBlob+i]; break; } default: @@ -967,7 +1030,7 @@ Module.postRun.push(function(namespace/*the module object, the target for if(null===val) { api.sqlite3_result_null(pCx); break; - }else if(isSupportedTypedArray(val)){ + }else if(isBindableTypedArray(val)){ const pBlob = api.wasm.mallocFromTypedArray(val); api.sqlite3_result_blob(pCx, pBlob, val.byteLength, api.SQLITE_TRANSIENT); @@ -1081,7 +1144,7 @@ Module.postRun.push(function(namespace/*the module object, the target for return t; default: //console.log("isSupportedBindType",t,v); - return isSupportedTypedArray(v) ? BindTypes.blob : undefined; + return isBindableTypedArray(v) ? BindTypes.blob : undefined; } }; @@ -1159,7 +1222,7 @@ Module.postRun.push(function(namespace/*the module object, the target for try{ const n = api.wasm.lengthBytesUTF8(val)+1/*required for NUL terminator*/; const pStr = api.wasm.stackAlloc(n); - api.wasm.stringToUTF8Array(val, api.wasm.HEAP8, pStr, n); + api.wasm.stringToUTF8Array(val, api.wasm.HEAPU8(), pStr, n); const f = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; return f(stmt._pStmt, ndx, pStr, n-1, api.SQLITE_TRANSIENT); }finally{ @@ -1168,7 +1231,7 @@ Module.postRun.push(function(namespace/*the module object, the target for }else{ const bytes = api.wasm.intArrayFromString(val,true); const pStr = api.wasm._malloc(bytes.length || 1); - api.wasm._malloc.HEAP.set(bytes.length ? bytes : [0], pStr); + api.wasm.HEAPU8().set(bytes.length ? bytes : [0], pStr); try{ const f = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; return f(stmt._pStmt, ndx, pStr, bytes.length, api.SQLITE_TRANSIENT); @@ -1205,7 +1268,7 @@ Module.postRun.push(function(namespace/*the module object, the target for case BindTypes.blob: { if('string'===typeof val){ rc = f._.string(stmt, ndx, val, true); - }else if(!isSupportedTypedArray(val)){ + }else if(!isBindableTypedArray(val)){ toss("Binding a value as a blob requires", "that it be a string, Uint8Array, or Int8Array."); }else if(1){ @@ -1213,7 +1276,7 @@ Module.postRun.push(function(namespace/*the module object, the target for const stack = api.wasm.stackSave(); try{ const pBlob = api.wasm.stackAlloc(val.byteLength || 1); - api.wasm.HEAP8.set(val.byteLength ? val : [0], pBlob) + api.wasm.HEAP8().set(val.byteLength ? val : [0], pBlob) rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, val.byteLength, api.SQLITE_TRANSIENT); }finally{ @@ -1379,7 +1442,7 @@ Module.postRun.push(function(namespace/*the module object, the target for return this; } else if('object'===typeof arg/*null was checked above*/ - && !isSupportedTypedArray(arg)){ + && !isBindableTypedArray(arg)){ /* Treat each property of arg as a named bound parameter. */ if(1!==arguments.length){ toss("When binding an object, an index argument is not permitted."); @@ -1501,10 +1564,11 @@ Module.postRun.push(function(namespace/*the module object, the target for case api.SQLITE_TEXT: return api.sqlite3_column_text(this._pStmt, ndx); case api.SQLITE_BLOB: { - const n = api.sqlite3_column_bytes(this._pStmt, ndx); - const ptr = api.sqlite3_column_blob(this._pStmt, ndx); - const rc = new Uint8Array(n); - for(let i = 0; i < n; ++i) rc[i] = api.wasm.HEAP8[ptr + i]; + const n = api.sqlite3_column_bytes(this._pStmt, ndx), + ptr = api.sqlite3_column_blob(this._pStmt, ndx), + rc = new Uint8Array(n), + heap = n ? api.wasm.HEAP8() : false; + for(let i = 0; i < n; ++i) rc[i] = heap[ptr + i]; if(n && this.db._blobXfer instanceof Array){ /* This is an optimization soley for the Worker-based API. These values will be diff --git a/ext/fiddle/testing1.js b/ext/fiddle/testing1.js index 44e285f73a..6cae6de8fa 100644 --- a/ext/fiddle/testing1.js +++ b/ext/fiddle/testing1.js @@ -190,17 +190,13 @@ }; const runTests = function(Module){ - T.assert(Module._free instanceof Function). - assert(Module.allocate instanceof Function). - assert(Module.addFunction instanceof Function). - assert(Module.removeFunction instanceof Function); const sqlite3 = Module.sqlite3; const api = sqlite3.api; const oo = sqlite3.SQLite3; log("Loaded module:",api.sqlite3_libversion(), api.sqlite3_sourceid()); log("Build options:",oo.compileOptionUsed()); - log("api.wasm.HEAP8 size =",api.wasm.HEAP8.length); + log("api.wasm.HEAP8 size =",api.wasm.HEAP8().length); log("wasmEnum",JSON.parse(Module.ccall('sqlite3_wasm_enum_json', 'string', []))); [ /* Spot-check a handful of constants to make sure they got installed... */ 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', @@ -208,6 +204,13 @@ ].forEach(function(k){ T.assert('number' === typeof api[k]); }); + [/* Spot-check a few of the WASM API methods. */ + '_free', '_malloc', 'addFunction', 'stackRestore' + ].forEach(function(k){ + T.assert(Module[k] instanceof Function). + assert(api.wasm[k] instanceof Function); + }); + const db = new oo.DB(); try { log("DB:",db.filename); diff --git a/ext/fiddle/wasm_util.c b/ext/fiddle/wasm_util.c index 54326b2977..91b8911934 100644 --- a/ext/fiddle/wasm_util.c +++ b/ext/fiddle/wasm_util.c @@ -12,9 +12,14 @@ ** *********************************************************************** ** -** Utility functions for use with the emscripten/WASM bits. These +** Utility functions for use with the emscripten/WASM bits. These ** functions ARE NOT part of the sqlite3 public API. They are strictly ** for internal use by the JS/WASM bindings. +** +** This file is intended to be WASM-compiled together with sqlite3.c, +** e.g.: +** +** emcc ... sqlite3.c wasm_util.c */ /** Result value of sqlite3_wasm_enum_json(). */ diff --git a/manifest b/manifest index 3697a203f1..cfc2a5ac5b 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C wasm:\sadded\sutility\sC\scode\sto\sgenerate\sa\sJSON-format\s"enum"\sof\sthe\snumerous\sSQLITE_xyz\sconstants\sso\sthat\swe\sdo\snot\srisk\sthose\sgetting\sout\sof\ssync\sin\sthe\sJS\scode.\sRenamed\sinitSqlite3Module\sto\ssqlite3InitModule.\sCleanups\sin\sthe\sTypedArray\shandling. -D 2022-06-25T10:30:24.409 +C wasm:\slots\sof\sdoc\sadditions\sand\srefactoring.\sRefactored\sthe\sWASM\smemory\sheap\susage\sto\shopefully\seventually\saccount\sfor\sa\sruntime-growable\sheap.\sDifferentiate\sbetween\ssupported\sTypedArray\stypes\sfor\sinput\sSQL\sstrings\svs\sbinding/fetching\sblobs.\sMight\s(untested)\shave\simplemented\sthe\sability\sto\sbind\sUtfNNArray\svalues\sas\sblobs,\swhere\sNN\sis\sone\sof\s16\sor\s32. +D 2022-06-25T18:18:45.580 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in b3ccd1a79e6364d49c465b38cec5eccdc74dfdc06501be62ef8eb01dc1f93f43 +F Makefile.in fff0e19d74fe31d6c2960e72b79a1c82aeb971b32fa5002ace2ceb906b9917e5 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc de7cb3e095ce2fdc33513ccd76ebdaeda1483d0ddab0410fe65cbdeadd4c0ee1 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -65,14 +65,14 @@ F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba83 F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf -F ext/fiddle/sqlite3-api.js 03ac065f4bc68eefd3b09cf3957a58891cbfcebe2a5ffe1c8a10f16a58dfc70b +F ext/fiddle/sqlite3-api.js 690921c0daabde10f7eea1fd1d2d4b034ef0d66d68292931403be0b8e6fccf6a F ext/fiddle/sqlite3-worker.js 50b7a9ce14c8fae0af965e35605fe12cafb79c1e01e99216d8110d8b02fbf4b5 F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a -F ext/fiddle/testing1.js f9615ff58b9de6879e4836618b34322085510ec44c6754e725a92a097c908a6f +F ext/fiddle/testing1.js b7e34d83d6cb2f640311654266cbbe85f4144ef31fda7615d6e91c6486d3890f F ext/fiddle/testing2.html 9063b2430ade2fe9da4e711addd1b51a2741cf0c7ebf6926472a5e5dd63c0bc4 F ext/fiddle/testing2.js 7b45b4e7fddbd51dbaf89b6722c02758051b34bac5a98c11b569a7e7572f88ee -F ext/fiddle/wasm_util.c b63e00c2f264ab4a9c45c9f9727627cbc4d8aa2f93c5dd09e8105d63ff7e0872 +F ext/fiddle/wasm_util.c dc9f6e8882b0777037b01d2c8ef2dd9360306a37980f6908cfa3909bf6c25da7 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P da1d3151a440567f34a2f6c0b2bfc2e9fab81c256cc361c9ce7b46f2c23a2aa8 -R ff53072845ce3be319c8a5687950f355 +P 778062e3b415dca5104eee398950741b6dbb9d4bdf7c998eef18371a42669946 +R 7173ecb48dc9d037c18c86459c1a9792 U stephan -Z 0ce9f393ac2f284b021257ae10338830 +Z d57bc6484b99c6c0083ee3c3d006ea5c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7a1e3598fa..94a0429076 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -778062e3b415dca5104eee398950741b6dbb9d4bdf7c998eef18371a42669946 \ No newline at end of file +e10d57dfbaa672a3a4cbfd9a9209552c3bde15cc75af838690ca412fd182066a \ No newline at end of file From 3691e5621003facb27e5ca6194b1726a4be94966 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Jun 2022 18:55:08 +0000 Subject: [PATCH 011/151] Fix documentation type. [forum:/forumpost/8d900996ed|forum post 8d900996ed]. FossilOrigin-Name: 869061f18d2f2f500451c87ab62d3ca71a5321d5246b2e2c7bf960e48c6b5250 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e662e1ccf3..384b913f49 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.39.0 -D 2022-06-25T14:57:57.276 +C Fix\sdocumentation\stype.\s\s[forum:/forumpost/8d900996ed|forum\spost\s8d900996ed]. +D 2022-06-25T18:55:08.426 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -572,7 +572,7 @@ F src/resolve.c 1655e44c77c51ebbe82924287528a78bd4a4aaaf34189dbae28d19ccf2ca615c F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 2db8bbf13f57cc9872f9b927546e98f5ea583e9e364182426492a434a994421b F src/shell.c.in 08e59f1cb9d9b1180aba52861aaada0c95f6ddd210488719684e160a0724c806 -F src/sqlite.h.in 172528c287399a34f188154017b7268bf82c6d5b780902e361958d2318c4e37c +F src/sqlite.h.in 01573eae96721f2a8ee2a9e3b7140ceeba2e9c44350911890b89b8ff0dcf6781 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d F src/sqliteInt.h 8353e96646372efdb0795a13cd9949831b4992c928de8f5c43b2524e8a4c6e7b @@ -1978,11 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cd6254fcd32798f7be4e6d827597ddaa2e46ac6e2f0149cd3a3be0416fa18835 -R 4b3e64a86ee9acaa4f97f1ab6ad6f3ae -T +sym-major-release * -T +sym-release * -T +sym-version-3.39.0 * +P 14e166f40dbfa6e055543f8301525f2ca2e96a02a57269818b9e69e162e98918 +R 3c13bbe66e273ed1005f5d2b6397dfab U drh -Z f64bfe891f96e530fe80668f0f22f84a +Z e781f3f92a677689bfc4c4e8aeba83ca # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 03a3b1ba34..9c0292a724 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14e166f40dbfa6e055543f8301525f2ca2e96a02a57269818b9e69e162e98918 \ No newline at end of file +869061f18d2f2f500451c87ab62d3ca71a5321d5246b2e2c7bf960e48c6b5250 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ca56b8cb98..e2281e4978 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6282,7 +6282,7 @@ sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** ** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name ** for the N-th database on database connection D, or a NULL pointer of N is -** out of range. An N alue of 0 means the main database file. An N of 1 is +** out of range. An N value of 0 means the main database file. An N of 1 is ** the "temp" schema. Larger values of N correspond to various ATTACH-ed ** databases. ** From f79d277999f7be162822fb61017ba81dcced1fc2 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 25 Jun 2022 19:07:58 +0000 Subject: [PATCH 012/151] wasm: added bindings for sqlite3_extended_result_codes(), sqlite3_open_v2(), and the SQLITE_OPEN_... flags. FossilOrigin-Name: ac876ab9913332c9a5e3045824bf92a2501707aecfb13906d5c280302d9878a9 --- ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api | 1 + ext/fiddle/sqlite3-api.js | 73 ++++++++++++++++++----- ext/fiddle/testing1.js | 6 +- ext/fiddle/wasm_util.c | 14 +++++ manifest | 18 +++--- manifest.uuid | 2 +- 6 files changed, 87 insertions(+), 27 deletions(-) diff --git a/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api b/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api index 59dfef45b6..dae46f0359 100644 --- a/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api @@ -26,6 +26,7 @@ _sqlite3_data_count _sqlite3_db_filename _sqlite3_errmsg _sqlite3_exec +_sqlite3_extended_result_codes _sqlite3_finalize _sqlite3_interrupt _sqlite3_libversion diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js index 5350464e90..0925f2cddd 100644 --- a/ext/fiddle/sqlite3-api.js +++ b/ext/fiddle/sqlite3-api.js @@ -162,11 +162,43 @@ Module.postRun.push(function(namespace/*the module object, the target for The main sqlite3 binding API gets installed into this object, mimicking the C API as closely as we can. The numerous members names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as - possible, identically to the C-native counterparts. A very few - exceptions may require an additional level of proxy function, as - documented in this object. + possible, identically to the C-native counterparts, as documented at: + + https://www.sqlite.org/c3ref/intro.html + + A very few exceptions require an additional level of proxy + function or may otherwise require special attention in the WASM + environment, and all such cases are document here. Those not + documented here are installed as 1-to-1 proxies for their C-side + counterparts. */ const api = { + /** + When using sqlite3_open_v2() it is important to keep the following + in mind: + + https://www.sqlite.org/c3ref/open.html + + - The flags for use with its 3rd argument are installed in this + object using the C-cide names, e.g. SQLITE_OPEN_CREATE. + + - If the combination of flags passed to it are invalid, + behavior is undefined. Thus is is never okay to call this + with fewer than 3 arguments, as JS will default the + missing arguments to `undefined`, which will result in a + flag value of 0. Most of the available SQLITE_OPEN_xxx + flags are meaningless in the WASM build, e.g. the mutext- + and cache-related flags, but they are retained in this + API for consistency's sake. + + - The final argument to this function specifies the VFS to + use, which is largely (but not entirely!) meaningless in + the WASM environment. It should always be null or + undefined, and it is safe to elide that argument when + calling this function. + */ + sqlite3_open_v2: function(filename,dbPtrPtr,flags,vfsStr){}/*installed later*/, + /** The sqlite3_prepare_v2() binding handles two different uses with differing JS/WASM semantics: @@ -204,11 +236,11 @@ Module.postRun.push(function(namespace/*the module object, the target for If sql is not a string or supported TypedArray, it must be a _pointer_ to a string which was allocated via api.wasm.allocateUTF8OnStack(), api.wasm._malloc(), or - equivalent. In that case, - the final argument may be 0/null/undefined or must be a - pointer to which the "tail" of the compiled SQL is written, - as documented for the C-side sqlite3_prepare_v2(). In case - (2), the underlying C function is called with: + equivalent. In that case, the final argument may be + 0/null/undefined or must be a pointer to which the "tail" + of the compiled SQL is written, as documented for the + C-side sqlite3_prepare_v2(). In case (2), the underlying C + function is called with: (pDb, sqlAsPointer, -1, ppStmt, pzTail) @@ -219,14 +251,14 @@ Module.postRun.push(function(namespace/*the module object, the target for sqlAsPointer value. If passed an invalid 2nd argument type, this function will - throw. That behaviour is in strong constrast to all of the + throw. That behavior is in strong constrast to all of the other C-bound functions (which return a non-0 result code on error) but is necessary because we have to way to set the db's error state such that this function could return a non-0 integer and the client could call sqlite3_errcode() or sqlite3_errmsg() to fetch it. */ - sqlite3_prepare_v2: undefined/*installed later*/, + sqlite3_prepare_v2: function(dbPtr, sql, sqlByteLen, stmtPtrPtr, strPtrPtr){}/*installed later*/, /** Holds state which are specific to the WASM-related @@ -345,12 +377,12 @@ Module.postRun.push(function(namespace/*the module object, the target for ["sqlite3_db_filename", "string", ["number", "string"]], ["sqlite3_errmsg", "string", ["number"]], ["sqlite3_exec", "number", ["number", "string", "number", "number", "number"]], + ["sqlite3_extended_result_codes", "number", ["number", "number"]], ["sqlite3_finalize", "number", ["number"]], ["sqlite3_interrupt", "void", ["number"]], ["sqlite3_libversion", "string", []], ["sqlite3_open", "number", ["string", "number"]], - //["sqlite3_open_v2", "number", ["string", "number", "number", "string"]], - //^^^^ TODO: add the flags needed for the 3rd arg + ["sqlite3_open_v2", "number", ["string", "number", "number", "string"]], /* sqlite3_prepare_v2() is handled separately due to us requiring two different sets of semantics for that function. */ ["sqlite3_reset", "number", ["number"]], @@ -406,8 +438,9 @@ Module.postRun.push(function(namespace/*the module object, the target for /* Import C-level constants... */ //console.log("wasmEnum=",SQM.ccall('sqlite3_wasm_enum_json', 'string', [])); const wasmEnum = JSON.parse(SQM.ccall('sqlite3_wasm_enum_json', 'string', [])); - ['resultCodes','dataTypes','udfFlags', - 'encodings','blobFinalizers'].forEach(function(t){ + ['blobFinalizers', 'dataTypes','encodings', + 'openFlags', 'resultCodes','udfFlags' + ].forEach(function(t){ Object.keys(wasmEnum[t]).forEach(function(k){ api[k] = wasmEnum[t][k]; }); @@ -493,6 +526,12 @@ Module.postRun.push(function(namespace/*the module object, the target for class, ":memory:" DBs are not supported (until/unless we can find a way to export those as well). The naming semantics will certainly evolve as this API does. + + The db is opened with a fixed set of flags: + (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXRESCODE). This API may change in the future + permit the caller to provide those flags via an additional + argument. */ const DB = function(arg){ let buffer, fn; @@ -526,7 +565,11 @@ Module.postRun.push(function(namespace/*the module object, the target for const stack = api.wasm.stackSave(); const ppDb = api.wasm.stackAlloc(4) /* output (sqlite3**) arg */; api.wasm.setValue(ppDb, 0, "i32"); - try {this.checkRc(api.sqlite3_open(fn, ppDb));} + try { + this.checkRc(api.sqlite3_open_v2(fn, ppDb, api.SQLITE_OPEN_READWRITE + | api.SQLITE_OPEN_CREATE + | api.SQLITE_OPEN_EXRESCODE, null)); + } finally{api.wasm.stackRestore(stack);} this._pDb = api.wasm.getValue(ppDb, "i32"); this.filename = fn; diff --git a/ext/fiddle/testing1.js b/ext/fiddle/testing1.js index 6cae6de8fa..552df5d6f7 100644 --- a/ext/fiddle/testing1.js +++ b/ext/fiddle/testing1.js @@ -27,7 +27,8 @@ const test1 = function(db,sqlite3){ const api = sqlite3.api; log("Basic sanity tests..."); - T.assert(db._pDb); + T.assert(db._pDb). + assert(0===api.sqlite3_extended_result_codes(db._pDb,1)); let st = db.prepare( new TextEncoder('utf-8').encode("select 3 as a") /* Testing handling of Uint8Array input */ @@ -200,7 +201,8 @@ log("wasmEnum",JSON.parse(Module.ccall('sqlite3_wasm_enum_json', 'string', []))); [ /* Spot-check a handful of constants to make sure they got installed... */ 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', - 'SQLITE_STATIC', 'SQLITE_DIRECTONLY' + 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', + 'SQLITE_OPEN_CREATE' ].forEach(function(k){ T.assert('number' === typeof api[k]); }); diff --git a/ext/fiddle/wasm_util.c b/ext/fiddle/wasm_util.c index 91b8911934..a02063fc48 100644 --- a/ext/fiddle/wasm_util.c +++ b/ext/fiddle/wasm_util.c @@ -103,6 +103,20 @@ const char * sqlite3_wasm_enum_json(void){ SD(SQLITE_DIRECTONLY); SDFinal(SQLITE_INNOCUOUS); + sqlite3_str_appendall(s,",\"openFlags\": {"); + /* Noting that not all of these will have any effect in WASM-space. */ + SD(SQLITE_OPEN_READONLY); + SD(SQLITE_OPEN_READWRITE); + SD(SQLITE_OPEN_CREATE); + SD(SQLITE_OPEN_URI); + SD(SQLITE_OPEN_MEMORY); + SD(SQLITE_OPEN_NOMUTEX); + SD(SQLITE_OPEN_FULLMUTEX); + SD(SQLITE_OPEN_SHAREDCACHE); + SD(SQLITE_OPEN_PRIVATECACHE); + SD(SQLITE_OPEN_EXRESCODE); + SDFinal(SQLITE_OPEN_NOFOLLOW); + #undef SD_ #undef SD #undef SDFinal diff --git a/manifest b/manifest index cfc2a5ac5b..63ba8de7e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm:\slots\sof\sdoc\sadditions\sand\srefactoring.\sRefactored\sthe\sWASM\smemory\sheap\susage\sto\shopefully\seventually\saccount\sfor\sa\sruntime-growable\sheap.\sDifferentiate\sbetween\ssupported\sTypedArray\stypes\sfor\sinput\sSQL\sstrings\svs\sbinding/fetching\sblobs.\sMight\s(untested)\shave\simplemented\sthe\sability\sto\sbind\sUtfNNArray\svalues\sas\sblobs,\swhere\sNN\sis\sone\sof\s16\sor\s32. -D 2022-06-25T18:18:45.580 +C wasm:\sadded\sbindings\sfor\ssqlite3_extended_result_codes(),\ssqlite3_open_v2(),\sand\sthe\sSQLITE_OPEN_...\sflags. +D 2022-06-25T19:07:58.321 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -56,7 +56,7 @@ F ext/expert/sqlite3expert.c 6ca30d73b9ed75bd56d6e0d7f2c962d2affaa72c505458619d0 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 F ext/fiddle/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 -F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api 2ef8a8d58815a96de196d085b8aaf126400b8d13a5b7f724f86dfe9f61d0600d +F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api 356c356931b58eccf68367120f304db43ab6c2ef2f62f17f12f5a99737b43c38 F ext/fiddle/EXPORTED_RUNTIME_METHODS a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 F ext/fiddle/Makefile e25d34a0e1324f771d64c09c592601b97219282011587e6ce410fa8acdedb913 F ext/fiddle/SqliteTestUtil.js 2e87d424b12674476bdf8139934dcacc3ff8a7a5f5ff4392ba5e5a8d8cee9fbd @@ -65,14 +65,14 @@ F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba83 F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf -F ext/fiddle/sqlite3-api.js 690921c0daabde10f7eea1fd1d2d4b034ef0d66d68292931403be0b8e6fccf6a +F ext/fiddle/sqlite3-api.js add7b016798bc77de39573e4d9f65a35d4a4e1dce0abe6a15da70d99c0c6994a F ext/fiddle/sqlite3-worker.js 50b7a9ce14c8fae0af965e35605fe12cafb79c1e01e99216d8110d8b02fbf4b5 F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a -F ext/fiddle/testing1.js b7e34d83d6cb2f640311654266cbbe85f4144ef31fda7615d6e91c6486d3890f +F ext/fiddle/testing1.js fbeac92a5ac22668a54fd358ffc75c275d83e505e770aa484045614cb2a6cf44 F ext/fiddle/testing2.html 9063b2430ade2fe9da4e711addd1b51a2741cf0c7ebf6926472a5e5dd63c0bc4 F ext/fiddle/testing2.js 7b45b4e7fddbd51dbaf89b6722c02758051b34bac5a98c11b569a7e7572f88ee -F ext/fiddle/wasm_util.c dc9f6e8882b0777037b01d2c8ef2dd9360306a37980f6908cfa3909bf6c25da7 +F ext/fiddle/wasm_util.c 5944e38a93d3a436a47dd7c69059844697b6a5e44499656baa6da0ffe7fe23d5 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 778062e3b415dca5104eee398950741b6dbb9d4bdf7c998eef18371a42669946 -R 7173ecb48dc9d037c18c86459c1a9792 +P e10d57dfbaa672a3a4cbfd9a9209552c3bde15cc75af838690ca412fd182066a +R 43f4d6e17f529d8fb5204f439ce0ffd6 U stephan -Z d57bc6484b99c6c0083ee3c3d006ea5c +Z d6d33ac3906437150708dfba4da65152 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 94a0429076..7fb96a0c45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e10d57dfbaa672a3a4cbfd9a9209552c3bde15cc75af838690ca412fd182066a \ No newline at end of file +ac876ab9913332c9a5e3045824bf92a2501707aecfb13906d5c280302d9878a9 \ No newline at end of file From 62f1b48638658444b140a4b256d7b90b2fe5a5c8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Jun 2022 19:30:07 +0000 Subject: [PATCH 013/151] Increase the version number to 3.40.0 so as to begin the next development cycle. FossilOrigin-Name: 4542e32302067f24bf71c77620e09663f47c3ddce703c21ffa82d09532ae1d51 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 4a8c3b9681..7e16c94210 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.39.0 +3.40.0 diff --git a/configure b/configure index c5d122423d..e9f135f87e 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.39.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.40.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.39.0' -PACKAGE_STRING='sqlite 3.39.0' +PACKAGE_VERSION='3.40.0' +PACKAGE_STRING='sqlite 3.40.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1468,7 +1468,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.39.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.40.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1533,7 +1533,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.39.0:";; + short | recursive ) echo "Configuration of sqlite 3.40.0:";; esac cat <<\_ACEOF @@ -1661,7 +1661,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.39.0 +sqlite configure 3.40.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2080,7 +2080,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.39.0, which was +It was created by sqlite $as_me 3.40.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -12390,7 +12390,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.39.0, which was +This file was extended by sqlite $as_me 3.40.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12456,7 +12456,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.39.0 +sqlite config.status 3.40.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/manifest b/manifest index 384b913f49..64c6186831 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sdocumentation\stype.\s\s[forum:/forumpost/8d900996ed|forum\spost\s8d900996ed]. -D 2022-06-25T18:55:08.426 +C Increase\sthe\sversion\snumber\sto\s3.40.0\sso\sas\sto\sbegin\sthe\snext\ndevelopment\scycle. +D 2022-06-25T19:30:07.437 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -7,7 +7,7 @@ F Makefile.in bccb0ed3f05fc41aee15da77c844c48b5da419cbb9af35b8a147536c9ad1c822 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc de7cb3e095ce2fdc33513ccd76ebdaeda1483d0ddab0410fe65cbdeadd4c0ee1 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e -F VERSION fa8e7d2d1cc962f9e14c6d410387cf75860ee139462763fda887c1be4261f824 +F VERSION 8868ddfa6e1eee218286021a94b3e22d13e550c76c72d878857547ca001de24a F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 @@ -34,7 +34,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure 00378d14cca0ce02fa9eca28fd494789c2eda7d10d86003a5f9a42f34b0af418 x +F configure f959db96f314b3b91b3d658eebbc0a96b9542f1265c4de97e885aedb6bdcead5 x F configure.ac 3ef6eeff4387585bfcab76b0c3f6e15a0618587bb90245dd5d44e4378141bb35 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14e166f40dbfa6e055543f8301525f2ca2e96a02a57269818b9e69e162e98918 -R 3c13bbe66e273ed1005f5d2b6397dfab +P 869061f18d2f2f500451c87ab62d3ca71a5321d5246b2e2c7bf960e48c6b5250 +R 92cb1c705f905531a519c6252d828d8f U drh -Z e781f3f92a677689bfc4c4e8aeba83ca +Z 35b10d4261739959f6f1dcc8128c2264 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9c0292a724..f5b50c7ce1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -869061f18d2f2f500451c87ab62d3ca71a5321d5246b2e2c7bf960e48c6b5250 \ No newline at end of file +4542e32302067f24bf71c77620e09663f47c3ddce703c21ffa82d09532ae1d51 \ No newline at end of file From 54f1fc4f940b0b2cbc38d5decf8b17c1d9959f6d Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Jun 2022 20:32:29 +0000 Subject: [PATCH 014/151] Put an ALWAYS() on an unreachable branch. FossilOrigin-Name: 58caa50a410b7eb0e68658ea1e606d75ea85cdae04e864270c932246ba990b5e --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 10c7a89c01..7bceca571d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sflattening\sof\sa\ssubquery\sthat\sis\sthe\sright\soperand\sof\sa\sLEFT\sJOIN\nin\san\saggregate\squery\sas\slong\sas\sthere\sis\sno\sGROUP\sBY\sclause.\s\s(The\sGROUP\sBY\nclause\swill\sinterfere\swith\sthe\soperation\sof\sthe\sTK_IF_NULL_ROW\sexpression\nnodes.) -D 2022-06-25T19:43:44.263 +C Put\san\sALWAYS()\son\san\sunreachable\sbranch. +D 2022-06-25T20:32:29.777 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 602fe229f32a96ceccae4f40824129669582096f7c355f53dbac156c9fecef23 F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 -F src/vdbe.c ba1018c513ee177486a386dd0d4f52ec62e6fbaba9ebc23f100677af5599ff26 +F src/vdbe.c 1266f3a4744224253dd74f0080014be8056b062c6f2f6a81e229fa0d306d4102 F src/vdbe.h 07641758ca8b4f4c6d81ea667ea167c541e6ece21f5574da11e3d21ec37e2662 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c 602610f1252d59cd69742f78a1e2f6fbae40a4b407f5506a6a7b869b0df08ff2 @@ -1978,9 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4542e32302067f24bf71c77620e09663f47c3ddce703c21ffa82d09532ae1d51 27f68e47320c751e3663507500c1c44f0b7f885f89c678fce6a35b1bc372dd64 -R 4695ba0a9eb0bfc0b34e8cb342c27d65 -T +closed 27f68e47320c751e3663507500c1c44f0b7f885f89c678fce6a35b1bc372dd64 +P 2cf373b10c9bc4cbc5fe63d0a6948011df7bbc2f40dc025c9349f875da782b88 +R 23ab54c38d6c741c18713aabfeb224f1 U drh -Z 487b1bcb2151096fdcfe32d425a680ba +Z 708e41bc037d720b3b5f9e0e89bbf5aa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0483f5bafe..4e2da8ead8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2cf373b10c9bc4cbc5fe63d0a6948011df7bbc2f40dc025c9349f875da782b88 \ No newline at end of file +58caa50a410b7eb0e68658ea1e606d75ea85cdae04e864270c932246ba990b5e \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index fe5b4767b9..5a6a8fbdc2 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2646,7 +2646,7 @@ case OP_IfNullRow: { /* jump */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - if( pC && pC->nullRow ){ + if( ALWAYS(pC) && pC->nullRow ){ sqlite3VdbeMemSetNull(aMem + pOp->p3); goto jump_to_p2; } From bc7180cdb362b0069fed60b0c207533122cc5ec2 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 25 Jun 2022 21:41:26 +0000 Subject: [PATCH 015/151] wasm: corrected the isInt32() check to account for negative values. FossilOrigin-Name: 7223f4bb588b6c759754ef26cbefbb172e767eaa80989b8f9ef104d4e8b9d856 --- ext/fiddle/Makefile | 2 +- ext/fiddle/sqlite3-api.js | 5 ++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ext/fiddle/Makefile b/ext/fiddle/Makefile index 093f9260f9..b274574d7f 100644 --- a/ext/fiddle/Makefile +++ b/ext/fiddle/Makefile @@ -18,7 +18,7 @@ fiddle_files = emscripten.css fiddle.html \ fiddle_remote ?= ifeq (,$(fiddle_remote)) ifneq (,$(wildcard /home/stephan)) - fiddle_remote = wh2:www/wh/sqlite3/. + fiddle_remote = wh:www/wh/sqlite3/. else ifneq (,$(wildcard /home/drh)) #fiddle_remote = if appropriate, add that user@host:/path here endif diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js index 0925f2cddd..01eb4c57fd 100644 --- a/ext/fiddle/sqlite3-api.js +++ b/ext/fiddle/sqlite3-api.js @@ -120,10 +120,9 @@ Module.postRun.push(function(namespace/*the module object, the target for throw new Error(Array.prototype.join.call(arguments, ' ')); }; - /** Returns true if n is a 32-bit (signed) integer, - else false. */ + /** Returns true if n is a 32-bit (signed) integer, else false. */ const isInt32 = function(n){ - return (n===(n|0) && n<0xFFFFFFFF) ? true : undefined; + return !!(n===(n|0) && n<=2147483647 && n>=-2147483648); }; /** Returns v if v appears to be a TypedArray, else false. */ diff --git a/manifest b/manifest index 63ba8de7e0..773decce4e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm:\sadded\sbindings\sfor\ssqlite3_extended_result_codes(),\ssqlite3_open_v2(),\sand\sthe\sSQLITE_OPEN_...\sflags. -D 2022-06-25T19:07:58.321 +C wasm:\scorrected\sthe\sisInt32()\scheck\sto\saccount\sfor\snegative\svalues. +D 2022-06-25T21:41:26.982 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -58,14 +58,14 @@ F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be F ext/fiddle/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api 356c356931b58eccf68367120f304db43ab6c2ef2f62f17f12f5a99737b43c38 F ext/fiddle/EXPORTED_RUNTIME_METHODS a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 -F ext/fiddle/Makefile e25d34a0e1324f771d64c09c592601b97219282011587e6ce410fa8acdedb913 +F ext/fiddle/Makefile 1d303ee6449be3bab67f4b1456bacf903eb1c5e6d40d3b391651d7e0879d891a F ext/fiddle/SqliteTestUtil.js 2e87d424b12674476bdf8139934dcacc3ff8a7a5f5ff4392ba5e5a8d8cee9fbd F ext/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf -F ext/fiddle/sqlite3-api.js add7b016798bc77de39573e4d9f65a35d4a4e1dce0abe6a15da70d99c0c6994a +F ext/fiddle/sqlite3-api.js 5a6cc120f3eeaab65e49bcdab234e83d83c67440e04bd97191bdc004ac0cda35 F ext/fiddle/sqlite3-worker.js 50b7a9ce14c8fae0af965e35605fe12cafb79c1e01e99216d8110d8b02fbf4b5 F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e10d57dfbaa672a3a4cbfd9a9209552c3bde15cc75af838690ca412fd182066a -R 43f4d6e17f529d8fb5204f439ce0ffd6 +P ac876ab9913332c9a5e3045824bf92a2501707aecfb13906d5c280302d9878a9 +R c0ca90334fea302fa7780f66e0b99cb5 U stephan -Z d6d33ac3906437150708dfba4da65152 +Z 5c1bc27cd9cb9a8a18904e79ae01eef4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7fb96a0c45..a93b8e8b14 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac876ab9913332c9a5e3045824bf92a2501707aecfb13906d5c280302d9878a9 \ No newline at end of file +7223f4bb588b6c759754ef26cbefbb172e767eaa80989b8f9ef104d4e8b9d856 \ No newline at end of file From b0b734d145589a12298e48e33e7feee648f2f0bf Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 27 Jun 2022 11:28:25 +0000 Subject: [PATCH 016/151] Fix a missing dependency for json.lo in Makefile.in FossilOrigin-Name: 65930a5c069e7274b945ce1aed0abb0edba3d4ab4e63916cc38c11cdef998926 --- Makefile.in | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile.in b/Makefile.in index da75b76a59..e8f104c2a6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -874,7 +874,7 @@ hash.lo: $(TOP)/src/hash.c $(HDR) insert.lo: $(TOP)/src/insert.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c -json.lo: $(TOP)/src/json.c +json.lo: $(TOP)/src/json.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/json.c legacy.lo: $(TOP)/src/legacy.c $(HDR) diff --git a/manifest b/manifest index 7bceca571d..c8bcb16661 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Put\san\sALWAYS()\son\san\sunreachable\sbranch. -D 2022-06-25T20:32:29.777 +C Fix\sa\smissing\sdependency\sfor\sjson.lo\sin\sMakefile.in +D 2022-06-27T11:28:25.255 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in bccb0ed3f05fc41aee15da77c844c48b5da419cbb9af35b8a147536c9ad1c822 +F Makefile.in 731cf154c61e7cd63a2707600fecde816cedc113bd9901723522aefc74ddb292 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc de7cb3e095ce2fdc33513ccd76ebdaeda1483d0ddab0410fe65cbdeadd4c0ee1 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2cf373b10c9bc4cbc5fe63d0a6948011df7bbc2f40dc025c9349f875da782b88 -R 23ab54c38d6c741c18713aabfeb224f1 +P 58caa50a410b7eb0e68658ea1e606d75ea85cdae04e864270c932246ba990b5e +R 75e666d95bc19960088c32e5514624f5 U drh -Z 708e41bc037d720b3b5f9e0e89bbf5aa +Z 7850a40777a5511a9d66baab30b628c4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4e2da8ead8..b537de9432 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58caa50a410b7eb0e68658ea1e606d75ea85cdae04e864270c932246ba990b5e \ No newline at end of file +65930a5c069e7274b945ce1aed0abb0edba3d4ab4e63916cc38c11cdef998926 \ No newline at end of file From e26f592ea5664ab0ec07ec89271a1342192e57e6 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 29 Jun 2022 15:16:48 +0000 Subject: [PATCH 017/151] Have the sqlite_stmt table buffer all data for the current scan within the xFilter method. FossilOrigin-Name: 84a91c255e3d77728820561f16bdd9a87b7ff42b5430a9e13f404dfc3365c716 --- ext/misc/stmt.c | 121 +++++++++++++++++++++++++++++++----------------- manifest | 14 +++--- manifest.uuid | 2 +- 3 files changed, 87 insertions(+), 50 deletions(-) diff --git a/ext/misc/stmt.c b/ext/misc/stmt.c index 876b0e5cb6..ac0ae6f0f6 100644 --- a/ext/misc/stmt.c +++ b/ext/misc/stmt.c @@ -30,6 +30,16 @@ SQLITE_EXTENSION_INIT1 #ifndef SQLITE_OMIT_VIRTUALTABLE + +#define STMT_NUM_INTEGER_COLUMN 10 +typedef struct StmtRow StmtRow; +struct StmtRow { + sqlite3_int64 iRowid; /* Rowid value */ + char *zSql; /* column "sql" */ + int aCol[STMT_NUM_INTEGER_COLUMN+1]; /* all other column values */ + StmtRow *pNext; /* Next row to return */ +}; + /* stmt_vtab is a subclass of sqlite3_vtab which will ** serve as the underlying representation of a stmt virtual table */ @@ -47,8 +57,7 @@ typedef struct stmt_cursor stmt_cursor; struct stmt_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ sqlite3 *db; /* Database connection for this cursor */ - sqlite3_stmt *pStmt; /* Statement cursor is currently pointing at */ - sqlite3_int64 iRowid; /* The rowid */ + StmtRow *pRow; /* Current row */ }; /* @@ -122,10 +131,21 @@ static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } +static void stmtCsrReset(stmt_cursor *pCur){ + StmtRow *pRow = 0; + StmtRow *pNext = 0; + for(pRow=pCur->pRow; pRow; pRow=pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + pCur->pRow = 0; +} + /* ** Destructor for a stmt_cursor. */ static int stmtClose(sqlite3_vtab_cursor *cur){ + stmtCsrReset((stmt_cursor*)cur); sqlite3_free(cur); return SQLITE_OK; } @@ -136,8 +156,9 @@ static int stmtClose(sqlite3_vtab_cursor *cur){ */ static int stmtNext(sqlite3_vtab_cursor *cur){ stmt_cursor *pCur = (stmt_cursor*)cur; - pCur->iRowid++; - pCur->pStmt = sqlite3_next_stmt(pCur->db, pCur->pStmt); + StmtRow *pNext = pCur->pRow->pNext; + sqlite3_free(pCur->pRow); + pCur->pRow = pNext; return SQLITE_OK; } @@ -151,39 +172,11 @@ static int stmtColumn( int i /* Which column to return */ ){ stmt_cursor *pCur = (stmt_cursor*)cur; - switch( i ){ - case STMT_COLUMN_SQL: { - sqlite3_result_text(ctx, sqlite3_sql(pCur->pStmt), -1, SQLITE_TRANSIENT); - break; - } - case STMT_COLUMN_NCOL: { - sqlite3_result_int(ctx, sqlite3_column_count(pCur->pStmt)); - break; - } - case STMT_COLUMN_RO: { - sqlite3_result_int(ctx, sqlite3_stmt_readonly(pCur->pStmt)); - break; - } - case STMT_COLUMN_BUSY: { - sqlite3_result_int(ctx, sqlite3_stmt_busy(pCur->pStmt)); - break; - } - default: { - assert( i==STMT_COLUMN_MEM ); - i = SQLITE_STMTSTATUS_MEMUSED + - STMT_COLUMN_NSCAN - SQLITE_STMTSTATUS_FULLSCAN_STEP; - /* Fall thru */ - } - case STMT_COLUMN_NSCAN: - case STMT_COLUMN_NSORT: - case STMT_COLUMN_NAIDX: - case STMT_COLUMN_NSTEP: - case STMT_COLUMN_REPREP: - case STMT_COLUMN_RUN: { - sqlite3_result_int(ctx, sqlite3_stmt_status(pCur->pStmt, - i-STMT_COLUMN_NSCAN+SQLITE_STMTSTATUS_FULLSCAN_STEP, 0)); - break; - } + StmtRow *pRow = pCur->pRow; + if( i==STMT_COLUMN_SQL ){ + sqlite3_result_text(ctx, pRow->zSql, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_int(ctx, pRow->aCol[i]); } return SQLITE_OK; } @@ -194,7 +187,7 @@ static int stmtColumn( */ static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ stmt_cursor *pCur = (stmt_cursor*)cur; - *pRowid = pCur->iRowid; + *pRowid = pCur->pRow->iRowid; return SQLITE_OK; } @@ -204,7 +197,7 @@ static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int stmtEof(sqlite3_vtab_cursor *cur){ stmt_cursor *pCur = (stmt_cursor*)cur; - return pCur->pStmt==0; + return pCur->pRow==0; } /* @@ -219,9 +212,53 @@ static int stmtFilter( int argc, sqlite3_value **argv ){ stmt_cursor *pCur = (stmt_cursor *)pVtabCursor; - pCur->pStmt = 0; - pCur->iRowid = 0; - return stmtNext(pVtabCursor); + sqlite3_stmt *p = 0; + sqlite3_int64 iRowid = 1; + StmtRow **ppRow = 0; + + stmtCsrReset(pCur); + ppRow = &pCur->pRow; + for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){ + const char *zSql = sqlite3_sql(p); + int nSql = zSql ? strlen(zSql)+1 : 0; + StmtRow *pNew = (StmtRow*)sqlite3_malloc(sizeof(StmtRow) + nSql); + + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(StmtRow)); + if( zSql ){ + pNew->zSql = (char*)&pNew[1]; + memcpy(pNew->zSql, zSql, nSql); + } + pNew->aCol[STMT_COLUMN_NCOL] = sqlite3_column_count(p); + pNew->aCol[STMT_COLUMN_RO] = sqlite3_stmt_readonly(p); + pNew->aCol[STMT_COLUMN_BUSY] = sqlite3_stmt_busy(p); + pNew->aCol[STMT_COLUMN_NSCAN] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_FULLSCAN_STEP, 0 + ); + pNew->aCol[STMT_COLUMN_NSORT] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_SORT, 0 + ); + pNew->aCol[STMT_COLUMN_NAIDX] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_AUTOINDEX, 0 + ); + pNew->aCol[STMT_COLUMN_NSTEP] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_VM_STEP, 0 + ); + pNew->aCol[STMT_COLUMN_REPREP] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_REPREPARE, 0 + ); + pNew->aCol[STMT_COLUMN_RUN] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_RUN, 0 + ); + pNew->aCol[STMT_COLUMN_MEM] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_MEMUSED, 0 + ); + pNew->iRowid = iRowid++; + *ppRow = pNew; + ppRow = &pNew->pNext; + } + + return SQLITE_OK; } /* diff --git a/manifest b/manifest index c8bcb16661..1cf49067fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smissing\sdependency\sfor\sjson.lo\sin\sMakefile.in -D 2022-06-27T11:28:25.255 +C Have\sthe\ssqlite_stmt\stable\sbuffer\sall\sdata\sfor\sthe\scurrent\sscan\swithin\sthe\sxFilter\smethod. +D 2022-06-29T15:16:48.039 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -344,7 +344,7 @@ F ext/misc/shathree.c 7b17615869a495659f1569ada1d8d3d21b4a24614f2746d93cc87ef7c0 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 94df9bbfa514a563c1484f684a2df3d128a2f7209a84ca3ca100c68a0163e29f F ext/misc/sqlar.c 0ace5d3c10fe736dc584bf1159a36b8e2e60fab309d310cd8a0eecd9036621b6 -F ext/misc/stmt.c 35063044a388ead95557e4b84b89c1b93accc2f1c6ddea3f9710e8486a7af94a +F ext/misc/stmt.c 72b23e1746bedcf6e36907e972383e7eb74940344da90b2965149739b55cc801 F ext/misc/templatevtab.c 8a16a91a5ceaccfcbd6aaaa56d46828806e460dd194965b3f77bf38f14b942c4 F ext/misc/totype.c fa4aedeb07f66169005dffa8de3b0a2b621779fd44f85c103228a42afa71853b F ext/misc/uint.c 053fed3bce2e89583afcd4bf804d75d659879bbcedac74d0fa9ed548839a030b @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 58caa50a410b7eb0e68658ea1e606d75ea85cdae04e864270c932246ba990b5e -R 75e666d95bc19960088c32e5514624f5 -U drh -Z 7850a40777a5511a9d66baab30b628c4 +P 65930a5c069e7274b945ce1aed0abb0edba3d4ab4e63916cc38c11cdef998926 +R 9025eb614ce3aeb665704ae99984368a +U dan +Z b363217bc0e676f1566caa9eb7479dc4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b537de9432..cd7ff70186 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65930a5c069e7274b945ce1aed0abb0edba3d4ab4e63916cc38c11cdef998926 \ No newline at end of file +84a91c255e3d77728820561f16bdd9a87b7ff42b5430a9e13f404dfc3365c716 \ No newline at end of file From d85e4ac347a49f50266ce77d8ef53a26c8f1cc37 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 30 Jun 2022 11:01:48 +0000 Subject: [PATCH 018/151] Add missing "finish_test" command to the end of test script merge1.test. FossilOrigin-Name: 13cb3f1e63ed1e906f820655645a4966f0cae140ac442177b6685637dcfd365a --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/merge1.test | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 1cf49067fa..9fdd27f061 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\ssqlite_stmt\stable\sbuffer\sall\sdata\sfor\sthe\scurrent\sscan\swithin\sthe\sxFilter\smethod. -D 2022-06-29T15:16:48.039 +C Add\smissing\s"finish_test"\scommand\sto\sthe\send\sof\stest\sscript\smerge1.test. +D 2022-06-30T11:01:48.271 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1244,7 +1244,7 @@ F test/memjournal2.test 89a4e0d1084170a281efa4d54c2677599f986f44227f98f7dfae2828 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 F test/memsubsys1.test 9e7555a22173b8f1c96c281ce289b338fcba2abe8b157f8798ca195bbf1d347e F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08 -F test/merge1.test 0ade470d77b689c4a64dc7f736527fcd893140bcafa70af8f7b98523cfb83f16 +F test/merge1.test 2de6d6ef8d25402764b1aab49d8f9d7f89208c89a6674e437f76de4c812157b8 F test/minmax.test fe638b55d77d2375531a8f549b338eafcd9adfbd2f72df37ed77d9b26ca0a71a F test/minmax2.test cf9311babb6f0518d04e42fd6a42c619531c4309a9dd790a2c4e9b3bc595e0de F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 65930a5c069e7274b945ce1aed0abb0edba3d4ab4e63916cc38c11cdef998926 -R 9025eb614ce3aeb665704ae99984368a +P 84a91c255e3d77728820561f16bdd9a87b7ff42b5430a9e13f404dfc3365c716 +R 2a5f2c7138401d3f0c12c136204a2064 U dan -Z b363217bc0e676f1566caa9eb7479dc4 +Z 442415903905a18bee84a4ba74343e75 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cd7ff70186..64dcc2d832 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84a91c255e3d77728820561f16bdd9a87b7ff42b5430a9e13f404dfc3365c716 \ No newline at end of file +13cb3f1e63ed1e906f820655645a4966f0cae140ac442177b6685637dcfd365a \ No newline at end of file diff --git a/test/merge1.test b/test/merge1.test index 36969e8fc8..7ec4dab108 100644 --- a/test/merge1.test +++ b/test/merge1.test @@ -141,3 +141,5 @@ do_eqp_test 111 { `--RIGHT `--SCAN generate_series VIRTUAL TABLE INDEX 23: } + +finish_test From 2b294b543f8dc7d39ace0480c3c86884f45b67e3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 30 Jun 2022 22:46:28 +0000 Subject: [PATCH 019/151] Small performance improvement to sqlite3_finalize(). FossilOrigin-Name: 8a6913b66cc5af354497044ccb849eb80d00d0799362475f1537a6999196895e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeapi.c | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9fdd27f061..0d6bd35309 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smissing\s"finish_test"\scommand\sto\sthe\send\sof\stest\sscript\smerge1.test. -D 2022-06-30T11:01:48.271 +C Small\sperformance\simprovement\sto\ssqlite3_finalize(). +D 2022-06-30T22:46:28.469 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -645,7 +645,7 @@ F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 F src/vdbe.c 1266f3a4744224253dd74f0080014be8056b062c6f2f6a81e229fa0d306d4102 F src/vdbe.h 07641758ca8b4f4c6d81ea667ea167c541e6ece21f5574da11e3d21ec37e2662 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da -F src/vdbeapi.c 602610f1252d59cd69742f78a1e2f6fbae40a4b407f5506a6a7b869b0df08ff2 +F src/vdbeapi.c d68267db6e6641994e17c70670c40fd67ceb2352e42188815ed8c05d4d6502cb F src/vdbeaux.c 328b866880e67526300aa2361c5e50beb3d97dd8746db37dd4aa0135aea60e4d F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 5ebf05c0182addedb1607ade848e1c83cef40981df94d1abfab0c59288c6064f @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 84a91c255e3d77728820561f16bdd9a87b7ff42b5430a9e13f404dfc3365c716 -R 2a5f2c7138401d3f0c12c136204a2064 -U dan -Z 442415903905a18bee84a4ba74343e75 +P 13cb3f1e63ed1e906f820655645a4966f0cae140ac442177b6685637dcfd365a +R 34d4d9bd240555c32fd6af81aad0f565 +U drh +Z 92cfa4f9af2c01d5c0a3aeb41d9df236 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 64dcc2d832..60de05d832 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13cb3f1e63ed1e906f820655645a4966f0cae140ac442177b6685637dcfd365a \ No newline at end of file +8a6913b66cc5af354497044ccb849eb80d00d0799362475f1537a6999196895e \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 4e719ede36..97bc744c30 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -108,7 +108,9 @@ int sqlite3_finalize(sqlite3_stmt *pStmt){ if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; sqlite3_mutex_enter(db->mutex); checkProfileCallback(db, v); - rc = sqlite3VdbeFinalize(v); + assert( v->eVdbeState>=VDBE_READY_STATE ); + rc = sqlite3VdbeReset(v); + sqlite3VdbeDelete(v); rc = sqlite3ApiExit(db, rc); sqlite3LeaveMutexAndCloseZombie(db); } From 064390b2b1d6358f6799b8ce014b5c44af213df0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 1 Jul 2022 19:42:12 +0000 Subject: [PATCH 020/151] Performance improvement in resolveP2Values(). Save over 1 million CPU cycles by omitting the loop termination condition and exiting when the OP_Init (always the first opcode in any VDBE program) is encountered. FossilOrigin-Name: bb179140c8abfd9d05d62380daba027bdeabbbafc720b7e36b99a36af806017e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 11 ++++++++--- tool/mkopcodeh.tcl | 1 + 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 0d6bd35309..fa18daa8e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\simprovement\sto\ssqlite3_finalize(). -D 2022-06-30T22:46:28.469 +C Performance\simprovement\sin\sresolveP2Values().\s\sSave\sover\s1\smillion\sCPU\ncycles\sby\somitting\sthe\sloop\stermination\scondition\sand\sexiting\swhen\sthe\nOP_Init\s(always\sthe\sfirst\sopcode\sin\sany\sVDBE\sprogram)\sis\sencountered. +D 2022-07-01T19:42:12.598 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/vdbe.c 1266f3a4744224253dd74f0080014be8056b062c6f2f6a81e229fa0d306d4102 F src/vdbe.h 07641758ca8b4f4c6d81ea667ea167c541e6ece21f5574da11e3d21ec37e2662 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c d68267db6e6641994e17c70670c40fd67ceb2352e42188815ed8c05d4d6502cb -F src/vdbeaux.c 328b866880e67526300aa2361c5e50beb3d97dd8746db37dd4aa0135aea60e4d +F src/vdbeaux.c a0259ac215b5a3c7583e88108eb57583e825983b81c8e5f885343d4f8b83aeda F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 5ebf05c0182addedb1607ade848e1c83cef40981df94d1abfab0c59288c6064f F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -1909,7 +1909,7 @@ F tool/mkctimec.tcl ac96a74f5e6d9dac672d5229f79c583d3357a50e7d098e473e6b2ce2f8ae F tool/mkkeywordhash.c 35bfc41adacc4aa6ef6fca7fd0c63e0ec0534b78daf4d0cfdebe398216bbffc3 F tool/mkmsvcmin.tcl 6ecab9fe22c2c8de4d82d4c46797bda3d2deac8e763885f5a38d0c44a895ab33 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef -F tool/mkopcodeh.tcl 5dab48c49a25452257494e9601702ab63adaba6bd54a9b382615fa52661c8f8c +F tool/mkopcodeh.tcl bcb2bd5affb545fd219ef0304c7978e2a356407ab723f45ec8569235892c1c3f F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa F tool/mkpragmatab.tcl bd07bd59d45d0f3448e123d6937e9811195f9908a51e09d774609883055bfd3d F tool/mkshellc.tcl df5d249617f9cc94d5c48eb0401673eb3f31f383ecbc54e8a13ca3dd97e89450 @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 13cb3f1e63ed1e906f820655645a4966f0cae140ac442177b6685637dcfd365a -R 34d4d9bd240555c32fd6af81aad0f565 +P 8a6913b66cc5af354497044ccb849eb80d00d0799362475f1537a6999196895e +R 5971b08b257db606bcd37261acc2841e U drh -Z 92cfa4f9af2c01d5c0a3aeb41d9df236 +Z 70d456a11395b6ccc3ce693c73ee9fef # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 60de05d832..280ad8d176 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a6913b66cc5af354497044ccb849eb80d00d0799362475f1537a6999196895e \ No newline at end of file +bb179140c8abfd9d05d62380daba027bdeabbbafc720b7e36b99a36af806017e \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 7c3be404ef..4b94d9dd14 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -806,8 +806,8 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ p->readOnly = 1; p->bIsReader = 0; pOp = &p->aOp[p->nOp-1]; - while(1){ - + assert( p->aOp[0].opcode==OP_Init ); + while( 1 /* Loop termates when it reaches the OP_Init opcode */ ){ /* Only JUMP opcodes and the short list of special opcodes in the switch ** below need to be considered. The mkopcodeh.tcl generator script groups ** all these opcodes together near the front of the opcode list. Skip @@ -836,6 +836,10 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ p->bIsReader = 1; break; } + case OP_Init: { + assert( pOp->p2>=0 ); + goto resolve_p2_values_loop_exit; + } #ifndef SQLITE_OMIT_VIRTUALTABLE case OP_VUpdate: { if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; @@ -868,9 +872,10 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode]&OPFLG_JUMP)==0 || pOp->p2>=0); } - if( pOp==p->aOp ) break; + assert( pOp>p->aOp ); pOp--; } +resolve_p2_values_loop_exit: if( aLabel ){ sqlite3DbFreeNN(p->db, pParse->aLabel); pParse->aLabel = 0; diff --git a/tool/mkopcodeh.tcl b/tool/mkopcodeh.tcl index 57c6920111..8b4e345c67 100644 --- a/tool/mkopcodeh.tcl +++ b/tool/mkopcodeh.tcl @@ -158,6 +158,7 @@ set rp2v_ops { OP_JournalMode OP_VUpdate OP_VFilter + OP_Init } # Assign the smallest values to opcodes that are processed by resolveP2Values() From 2c1b1ddc55377d76d5d33f28e562b3250fc8a2d0 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 1 Jul 2022 21:03:19 +0000 Subject: [PATCH 021/151] Do not attempt the OP_Count optimization on queries with HAVING clauses. FossilOrigin-Name: 566b7842ee286ed8620a87b06de65bc173ad0d208df4157292238fcb059cb484 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 1 + test/select3.test | 8 ++++++++ 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index fa18daa8e5..de5732840a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovement\sin\sresolveP2Values().\s\sSave\sover\s1\smillion\sCPU\ncycles\sby\somitting\sthe\sloop\stermination\scondition\sand\sexiting\swhen\sthe\nOP_Init\s(always\sthe\sfirst\sopcode\sin\sany\sVDBE\sprogram)\sis\sencountered. -D 2022-07-01T19:42:12.598 +C Do\snot\sattempt\sthe\sOP_Count\soptimization\son\squeries\swith\sHAVING\sclauses. +D 2022-07-01T21:03:19.187 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -570,7 +570,7 @@ F src/printf.c 6166a30417b05c5b2f82e1f183f75faa2926ad60531c0b688a57dbc951441a20 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c 1655e44c77c51ebbe82924287528a78bd4a4aaaf34189dbae28d19ccf2ca615c F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c aa89ca7cf93fc5aa8df0955b068a57019ddc83e58c3a2133f7b4ca19ba8aae7d +F src/select.c fee45b7866181d86cb4e666adf148061166fbb4eb4df72ebc756901d48ba4025 F src/shell.c.in 08e59f1cb9d9b1180aba52861aaada0c95f6ddd210488719684e160a0724c806 F src/sqlite.h.in 01573eae96721f2a8ee2a9e3b7140ceeba2e9c44350911890b89b8ff0dcf6781 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1390,7 +1390,7 @@ F test/securedel.test 2f70b2449186a1921bd01ec9da407fbfa98c3a7a5521854c300c194b2f F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/select1.test 692e84cfa29c405854c69e8a4027183d64c22952866a123fabbce741a379e889 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 -F test/select3.test 399a0b23f8618bfe07ea24fb67f2952f620d6eba662bccabf6f4f71c2d89e6c9 +F test/select3.test ce4f78bbc809b0513f960f1ee84cdbc5af50ba112c343d5266558a8b2468f656 F test/select4.test f0684d3da3bccacbe2a1ebadf6fb49d9df6f53acb4c6ebc228a88d0d6054cc7b F test/select5.test 8afc5e5dcdebc2be54472e73ebd9cd1adef1225fd15d37a1c62f969159f390ae F test/select6.test 9b2fb4ffedf52e1b5703cfcae1212e7a4a063f014c0458d78d29aca3db766d1f @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8a6913b66cc5af354497044ccb849eb80d00d0799362475f1537a6999196895e -R 5971b08b257db606bcd37261acc2841e -U drh -Z 70d456a11395b6ccc3ce693c73ee9fef +P bb179140c8abfd9d05d62380daba027bdeabbbafc720b7e36b99a36af806017e +R d6af139cc8944872e222702884be1325 +U dan +Z 37caff9d5f68a04edba51ec8d81628bc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 280ad8d176..6e9ff26569 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb179140c8abfd9d05d62380daba027bdeabbbafc720b7e36b99a36af806017e \ No newline at end of file +566b7842ee286ed8620a87b06de65bc173ad0d208df4157292238fcb059cb484 \ No newline at end of file diff --git a/src/select.c b/src/select.c index fa18372ce6..fcd8247d22 100644 --- a/src/select.c +++ b/src/select.c @@ -5193,6 +5193,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect || pAggInfo->nFunc!=1 + || p->pHaving ){ return 0; } diff --git a/test/select3.test b/test/select3.test index 690514b496..809b549028 100644 --- a/test/select3.test +++ b/test/select3.test @@ -118,10 +118,18 @@ do_test select3-2.14 { } {1 {near ";": syntax error}} # Cannot have a HAVING without a GROUP BY +# +# Update: As of 3.39.0, you can. # do_execsql_test select3-3.1 { SELECT log, count(*) FROM t1 HAVING log>=4 } {} +do_execsql_test select3-3.2 { + SELECT count(*) FROM t1 HAVING log>=4 +} {} +do_execsql_test select3-3.3 { + SELECT count(*) FROM t1 HAVING log!=400 +} {31} # Toss in some HAVING clauses # From 20e9cbe0ff41b28e2e050c63490c4fcdb7b57c64 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 3 Jul 2022 11:12:59 +0000 Subject: [PATCH 022/151] Remove debugging code that has always been commented out. Cosmetic change. FossilOrigin-Name: b54064170c51a2c905f98737c224e19c06dc728342ca2224e32ebeb47f803bcd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 6 ------ 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index de5732840a..44e61891ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sattempt\sthe\sOP_Count\soptimization\son\squeries\swith\sHAVING\sclauses. -D 2022-07-01T21:03:19.187 +C Remove\sdebugging\scode\sthat\shas\salways\sbeen\scommented\sout.\s\sCosmetic\schange. +D 2022-07-03T11:12:59.625 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c b5a74c39d3123dadd190019c0afadd7f62b43dfe0712a39e9d0ff2fe26be5d93 +F src/btree.c d4037518cecba5f9f69be57800c71eb798848b82cb6940511fdf8c7af726a70a F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c 23f874642825d7eaaeeb7a3281b2b1a75e1d4c4dd9ae4dceddcd908266634214 @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb179140c8abfd9d05d62380daba027bdeabbbafc720b7e36b99a36af806017e -R d6af139cc8944872e222702884be1325 -U dan -Z 37caff9d5f68a04edba51ec8d81628bc +P 566b7842ee286ed8620a87b06de65bc173ad0d208df4157292238fcb059cb484 +R d49750ad5ea4a17f2067ad4595d4dbf5 +U drh +Z dbcc3e99f423baf35317581d02c84c1e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6e9ff26569..04676d57d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -566b7842ee286ed8620a87b06de65bc173ad0d208df4157292238fcb059cb484 \ No newline at end of file +b54064170c51a2c905f98737c224e19c06dc728342ca2224e32ebeb47f803bcd \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index facb951d3c..a9a4be99fc 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7015,12 +7015,6 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ assert( pPage->pBt->usableSize > (u32)(ptr-data) ); pc = get2byte(ptr); hdr = pPage->hdrOffset; -#if 0 /* Not required. Omit for efficiency */ - if( pcnCell*2 ){ - *pRC = SQLITE_CORRUPT_BKPT; - return; - } -#endif testcase( pc==(u32)get2byte(&data[hdr+5]) ); testcase( pc+sz==pPage->pBt->usableSize ); if( pc+sz > pPage->pBt->usableSize ){ From a57ac0a82738d5da8177c276f0904fe0089fad81 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 3 Jul 2022 11:16:03 +0000 Subject: [PATCH 023/151] Improved comment on sqlite3VdbeSwap(). No changes to code. FossilOrigin-Name: 6a8e4fb72a9e3dea9e5752c3d54fddba8878b355bd43f3c879f042f247a6610f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 9 ++++++++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 44e61891ed..de36ae8cec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sdebugging\scode\sthat\shas\salways\sbeen\scommented\sout.\s\sCosmetic\schange. -D 2022-07-03T11:12:59.625 +C Improved\scomment\son\ssqlite3VdbeSwap().\s\sNo\schanges\sto\scode. +D 2022-07-03T11:16:03.854 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/vdbe.c 1266f3a4744224253dd74f0080014be8056b062c6f2f6a81e229fa0d306d4102 F src/vdbe.h 07641758ca8b4f4c6d81ea667ea167c541e6ece21f5574da11e3d21ec37e2662 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c d68267db6e6641994e17c70670c40fd67ceb2352e42188815ed8c05d4d6502cb -F src/vdbeaux.c a0259ac215b5a3c7583e88108eb57583e825983b81c8e5f885343d4f8b83aeda +F src/vdbeaux.c 444c399df547e003be52cc51b460fed3b63e1f18939e6b773ff99c584954b726 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 5ebf05c0182addedb1607ade848e1c83cef40981df94d1abfab0c59288c6064f F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 566b7842ee286ed8620a87b06de65bc173ad0d208df4157292238fcb059cb484 -R d49750ad5ea4a17f2067ad4595d4dbf5 +P b54064170c51a2c905f98737c224e19c06dc728342ca2224e32ebeb47f803bcd +R 99d17d4f153cc68e746fb72ad8ba0840 U drh -Z dbcc3e99f423baf35317581d02c84c1e +Z bc5e9503636b3fa144917e41e898a5a3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 04676d57d4..8ec487544d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b54064170c51a2c905f98737c224e19c06dc728342ca2224e32ebeb47f803bcd \ No newline at end of file +6a8e4fb72a9e3dea9e5752c3d54fddba8878b355bd43f3c879f042f247a6610f \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 4b94d9dd14..4666f728ab 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -115,7 +115,14 @@ int sqlite3VdbeUsesDoubleQuotedString( #endif /* -** Swap all content between two VDBE structures. +** Swap byte-code between two VDBE structures. +** +** This happens after pB was previously run and returned +** SQLITE_SCHEMA. The statement was then reprepared in pA. +** This routine transfers the new bytecode in pA over to pB +** so that pB can be run again. The old pB byte code is +** moved back to pA so that it will be cleaned up when pA is +** finalized. */ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ Vdbe tmp, *pTmp; From f28727f61e270ecbd58e52318d3895990ff66fe5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 3 Jul 2022 14:25:47 +0000 Subject: [PATCH 024/151] Fix the initial-prefix optimization for the REGEXP extension such that it works even if the prefix contains characters that require a 3-byte UTF8 encoding. This should fix the problem reported by [forum:/forumpost/96692f8ba5|forum post 96692f8ba5]. FossilOrigin-Name: c94595a6e15490b432f099fefbe2429fa19287f7bdc86332cba0fd1e08f65bd6 --- ext/misc/regexp.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- test/regexp1.test | 22 ++++++++++++++++++++++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index b626ca424a..52973cc73f 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -685,7 +685,7 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6)); pRe->zInit[j++] = 0x80 | (x&0x3f); }else if( x<=0xffff ){ - pRe->zInit[j++] = (unsigned char)(0xd0 | (x>>12)); + pRe->zInit[j++] = (unsigned char)(0xe0 | (x>>12)); pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f); pRe->zInit[j++] = 0x80 | (x&0x3f); }else{ diff --git a/manifest b/manifest index de36ae8cec..018a75e20b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomment\son\ssqlite3VdbeSwap().\s\sNo\schanges\sto\scode. -D 2022-07-03T11:16:03.854 +C Fix\sthe\sinitial-prefix\soptimization\sfor\sthe\sREGEXP\sextension\ssuch\sthat\sit\nworks\seven\sif\sthe\sprefix\scontains\scharacters\sthat\srequire\sa\s3-byte\sUTF8\nencoding.\s\sThis\sshould\sfix\sthe\sproblem\sreported\sby\n[forum:/forumpost/96692f8ba5|forum\spost\s96692f8ba5]. +D 2022-07-03T14:25:47.812 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -334,7 +334,7 @@ F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 F ext/misc/prefixes.c 0f4f8cff5aebc00a7e3ac4021fd59cfe1a8e17c800ceaf592859ecb9cbc38196 F ext/misc/qpvtab.c 09738419e25f603a35c0ac8bd0a04daab794f48d08a9bc07a6085b9057b99009 -F ext/misc/regexp.c b267fd05ff8d38b22f4c2809d7b7a2c61d522e9faf2feb928dbb9662e4a3a386 +F ext/misc/regexp.c 03e483711534c437b2e29648d2a4b7730f5cb781a434ac8150907376bc4489f6 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 @@ -1342,7 +1342,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/recover.test ccb8c2623902a92ebb76770edd075cb4f75a4760bb7afde38026572c6e79070d -F test/regexp1.test 0c3ff80f66b0eff80e623eb5db7a3dad512095c573d78ac23009785f6d8f51ce +F test/regexp1.test 4a44e014664a109bbb1c37d29d9b61ca5aa5a7f49cc564c95208a80f818e3377 F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d F test/releasetest_data.tcl 11ba48a21ed1c808147b0e77c6e93d204577f4327ffe6d7c3b34cd3c01eac3a2 @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b54064170c51a2c905f98737c224e19c06dc728342ca2224e32ebeb47f803bcd -R 99d17d4f153cc68e746fb72ad8ba0840 +P 6a8e4fb72a9e3dea9e5752c3d54fddba8878b355bd43f3c879f042f247a6610f +R 3b0d1d13c75a6aec06b5d58ce72f012e U drh -Z bc5e9503636b3fa144917e41e898a5a3 +Z 9a117fdde370084441c6305b15600e26 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8ec487544d..141113cd80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a8e4fb72a9e3dea9e5752c3d54fddba8878b355bd43f3c879f042f247a6610f \ No newline at end of file +c94595a6e15490b432f099fefbe2429fa19287f7bdc86332cba0fd1e08f65bd6 \ No newline at end of file diff --git a/test/regexp1.test b/test/regexp1.test index 1eb56c672c..569dd66c2a 100644 --- a/test/regexp1.test +++ b/test/regexp1.test @@ -239,4 +239,26 @@ do_execsql_test regexp1-2.22 { SELECT 'abc$¢€xyz' REGEXP '^abc[^\u0025-X][^ -\u007f][^\u20ab]xyz$' } {1} +# 2022-07-03 +# https://sqlite.org/forum/forumpost/96692f8ba5 +# The REGEXP extension mishandles the prefix search optimization when +# the prefix contains 3-byte UTF8 characters. +# +reset_db +load_static_extension db regexp +do_execsql_test regexp1-3.1 { + CREATE TABLE t1(id INTEGER PRIMARY KEY, a TEXT); + INSERT INTO t1(id, a) VALUES(1, '日本語'); + SELECT a, hex(a), length(a) FROM t1; +} {日本語 E697A5E69CACE8AA9E 3} +do_execsql_test regexp1-3.2 { + SELECT * FROM t1 WHERE a='日本語'; +} {1 日本語} +do_execsql_test regexp1-3.3 { + SELECT * FROM t1 WHERE a LIKE '日本語'; +} {1 日本語} +do_execsql_test regexp1-3.4 { + SELECT * FROM t1 wHERE a REGEXP '日本語'; +} {1 日本語} + finish_test From 0e4ab0db7ee162e5d73437d3650078c145a68839 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 3 Jul 2022 18:12:43 +0000 Subject: [PATCH 025/151] Enhance the REGEXP extension so that the end-of-input indicate ("$") is allowed to occur on one branch of an OR ("|"). [forum:/forumpost/0107d5d40dd273e2|Forum post 0107d5d40dd273e2], second issue. FossilOrigin-Name: 3c04d21e6c632feb3bea8d1fa76bedcbfe254b0dc59865633d158a3f1bddefba --- ext/misc/regexp.c | 15 ++++++++------- manifest | 14 +++++++------- manifest.uuid | 2 +- test/regexp1.test | 19 +++++++++++++++++++ 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index 52973cc73f..d5b3872058 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -328,7 +328,9 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ } } for(i=0; inState; i++){ - if( pRe->aOp[pNext->aState[i]]==RE_OP_ACCEPT ){ rc = 1; break; } + int x = pNext->aState[i]; + while( pRe->aOp[x]==RE_OP_GOTO ) x += pRe->aArg[x]; + if( pRe->aOp[x]==RE_OP_ACCEPT ){ rc = 1; break; } } re_match_end: sqlite3_free(pToFree); @@ -483,7 +485,6 @@ static const char *re_subcompile_string(ReCompiled *p){ iStart = p->nState; switch( c ){ case '|': - case '$': case ')': { p->sIn.i--; return 0; @@ -520,6 +521,10 @@ static const char *re_subcompile_string(ReCompiled *p){ re_insert(p, iPrev, RE_OP_FORK, p->nState - iPrev+1); break; } + case '$': { + re_append(p, RE_OP_MATCH, RE_EOF); + break; + } case '{': { int m = 0, n = 0; int sz, j; @@ -656,11 +661,7 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ re_free(pRe); return zErr; } - if( rePeek(pRe)=='$' && pRe->sIn.i+1>=pRe->sIn.mx ){ - re_append(pRe, RE_OP_MATCH, RE_EOF); - re_append(pRe, RE_OP_ACCEPT, 0); - *ppRe = pRe; - }else if( pRe->sIn.i>=pRe->sIn.mx ){ + if( pRe->sIn.i>=pRe->sIn.mx ){ re_append(pRe, RE_OP_ACCEPT, 0); *ppRe = pRe; }else{ diff --git a/manifest b/manifest index 018a75e20b..64f106cefe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sinitial-prefix\soptimization\sfor\sthe\sREGEXP\sextension\ssuch\sthat\sit\nworks\seven\sif\sthe\sprefix\scontains\scharacters\sthat\srequire\sa\s3-byte\sUTF8\nencoding.\s\sThis\sshould\sfix\sthe\sproblem\sreported\sby\n[forum:/forumpost/96692f8ba5|forum\spost\s96692f8ba5]. -D 2022-07-03T14:25:47.812 +C Enhance\sthe\sREGEXP\sextension\sso\sthat\sthe\send-of-input\sindicate\s("$")\sis\nallowed\sto\soccur\son\sone\sbranch\sof\san\sOR\s("|").\n[forum:/forumpost/0107d5d40dd273e2|Forum\spost\s0107d5d40dd273e2],\ssecond\nissue. +D 2022-07-03T18:12:43.089 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -334,7 +334,7 @@ F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 F ext/misc/prefixes.c 0f4f8cff5aebc00a7e3ac4021fd59cfe1a8e17c800ceaf592859ecb9cbc38196 F ext/misc/qpvtab.c 09738419e25f603a35c0ac8bd0a04daab794f48d08a9bc07a6085b9057b99009 -F ext/misc/regexp.c 03e483711534c437b2e29648d2a4b7730f5cb781a434ac8150907376bc4489f6 +F ext/misc/regexp.c c1fb4f0639ad70f180fa7ebca3b9830c1c648cdcd072e28747f727699f3a5071 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 @@ -1342,7 +1342,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/recover.test ccb8c2623902a92ebb76770edd075cb4f75a4760bb7afde38026572c6e79070d -F test/regexp1.test 4a44e014664a109bbb1c37d29d9b61ca5aa5a7f49cc564c95208a80f818e3377 +F test/regexp1.test 611adedda9ab00c86a14e2a4becf4086ed712e83d86f576a884d21795213f16b F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d F test/releasetest_data.tcl 11ba48a21ed1c808147b0e77c6e93d204577f4327ffe6d7c3b34cd3c01eac3a2 @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6a8e4fb72a9e3dea9e5752c3d54fddba8878b355bd43f3c879f042f247a6610f -R 3b0d1d13c75a6aec06b5d58ce72f012e +P c94595a6e15490b432f099fefbe2429fa19287f7bdc86332cba0fd1e08f65bd6 +R 5a847080949684e34540bd8fbb3bcdb5 U drh -Z 9a117fdde370084441c6305b15600e26 +Z 843de43aa11ad4fe844e55becdb579a4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 141113cd80..7ce3a33e2a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c94595a6e15490b432f099fefbe2429fa19287f7bdc86332cba0fd1e08f65bd6 \ No newline at end of file +3c04d21e6c632feb3bea8d1fa76bedcbfe254b0dc59865633d158a3f1bddefba \ No newline at end of file diff --git a/test/regexp1.test b/test/regexp1.test index 569dd66c2a..fa941af53b 100644 --- a/test/regexp1.test +++ b/test/regexp1.test @@ -261,4 +261,23 @@ do_execsql_test regexp1-3.4 { SELECT * FROM t1 wHERE a REGEXP '日本語'; } {1 日本語} +# 2022-07-03 +# https://sqlite.org/forum/forumpost/96692f8ba5 Issue #2 +# The '$' token in REGEXP contained within other elements. +# +do_execsql_test regexp1-4.1 {SELECT 'xab' REGEXP 'a(b$|cd)';} {1} +do_execsql_test regexp1-4.1b {SELECT 'xab' REGEXP '(b$|cd)';} {1} +do_execsql_test regexp1-4.2 {SELECT 'xaby' REGEXP 'a(b$|cd)';} {0} +do_execsql_test regexp1-4.3 {SELECT 'xacd' REGEXP 'a(b$|cd)';} {1} +do_execsql_test regexp1-4.4 {SELECT 'xacdy' REGEXP 'a(b$|cd)';} {1} +do_execsql_test regexp1-4.5 {SELECT 'xab' REGEXP 'a(cd|b$)';} {1} +do_execsql_test regexp1-4.6 {SELECT 'xaby' REGEXP 'a(cd|b$)';} {0} +do_execsql_test regexp1-4.7 {SELECT 'xacd' REGEXP 'a(cd|b$)';} {1} +do_execsql_test regexp1-4.8 {SELECT 'xacdy' REGEXP 'a(cd|b$)';} {1} +do_execsql_test regexp1-4.9 {SELECT 'xab' REGEXP 'a(cd|b$|e)';} {1} +do_execsql_test regexp1-4.10 {SELECT 'xaby' REGEXP 'a(cd|b$|e)';} {0} +do_execsql_test regexp1-4.11 {SELECT 'xacd' REGEXP 'a(cd|b$|e)';} {1} +do_execsql_test regexp1-4.12 {SELECT 'xacdy' REGEXP 'a(cd|b$|e)';} {1} + + finish_test From 16316f1b68de0f6778c067b5e0f0947b46a2bcdb Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 4 Jul 2022 09:41:44 +0000 Subject: [PATCH 026/151] In the getNormalPage() routine of pager.c, consolidate pgno error checking into a single spot for small size reduction and performance increase. FossilOrigin-Name: a1c090e08139f99d30aa89db0756dc59fe8990ce15b3db4d4b726cc6acdab46f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 64f106cefe..5fa3a502bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sREGEXP\sextension\sso\sthat\sthe\send-of-input\sindicate\s("$")\sis\nallowed\sto\soccur\son\sone\sbranch\sof\san\sOR\s("|").\n[forum:/forumpost/0107d5d40dd273e2|Forum\spost\s0107d5d40dd273e2],\ssecond\nissue. -D 2022-07-03T18:12:43.089 +C In\sthe\sgetNormalPage()\sroutine\sof\spager.c,\sconsolidate\spgno\serror\schecking\ninto\sa\ssingle\sspot\sfor\ssmall\ssize\sreduction\sand\sperformance\sincrease. +D 2022-07-04T09:41:44.103 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -557,7 +557,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 2df2b33db88f00af13805d4573ee126bc5973f9e3b91d03c575fa7ba64e7dc41 F src/os_win.c a8ea80037e81127ca01959daa87387cc135f325c88dc745376c4f760de852a10 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 74596fc3d5d8a50de32c37225fc300cccd5ea27ea303c5f4b845d6572f999c5f +F src/pager.c a7cad005d788957737cb6662960f022d767dded35bcecf06dae090213d62e924 F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564 F src/pcache.c 084e638432c610f95aea72b8509f0845d2791293f39d1b82f0c0a7e089c3bb6b @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c94595a6e15490b432f099fefbe2429fa19287f7bdc86332cba0fd1e08f65bd6 -R 5a847080949684e34540bd8fbb3bcdb5 +P 3c04d21e6c632feb3bea8d1fa76bedcbfe254b0dc59865633d158a3f1bddefba +R 14e47e4bb1ab8f4db4305d42d7534790 U drh -Z 843de43aa11ad4fe844e55becdb579a4 +Z 5c670065e3a48de39ba2455ceb117563 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7ce3a33e2a..9df82a5fbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c04d21e6c632feb3bea8d1fa76bedcbfe254b0dc59865633d158a3f1bddefba \ No newline at end of file +a1c090e08139f99d30aa89db0756dc59fe8990ce15b3db4d4b726cc6acdab46f \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 95e6eb8af1..82e9dc7a69 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5495,7 +5495,6 @@ static int getPageNormal( assert( assert_pager_state(pPager) ); assert( pPager->hasHeldSharedLock==1 ); - if( pgno==0 ) return SQLITE_CORRUPT_BKPT; pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); if( pBase==0 ){ pPg = 0; @@ -5526,7 +5525,7 @@ static int getPageNormal( ** (*) obsolete. Was: maximum page number is 2^31 ** (2) Never try to fetch the locking page */ - if( pgno==PAGER_SJ_PGNO(pPager) ){ + if( pgno==0 || pgno==PAGER_SJ_PGNO(pPager) ){ rc = SQLITE_CORRUPT_BKPT; goto pager_acquire_err; } From 3325a5c4979ff5ee5886c0b615b6d2164eaf34a7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 4 Jul 2022 15:14:25 +0000 Subject: [PATCH 027/151] Size reduction and performance optimization in sqlite3WalFindFrame(). FossilOrigin-Name: 1a8c2e54375ee2cf73773b798fed0ae07b42f5e068fddc513c093de5c1f46615 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 12 +++++------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 5fa3a502bc..41d014f90b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sgetNormalPage()\sroutine\sof\spager.c,\sconsolidate\spgno\serror\schecking\ninto\sa\ssingle\sspot\sfor\ssmall\ssize\sreduction\sand\sperformance\sincrease. -D 2022-07-04T09:41:44.103 +C Size\sreduction\sand\sperformance\soptimization\sin\ssqlite3WalFindFrame(). +D 2022-07-04T15:14:25.563 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -654,7 +654,7 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8 F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c F src/vtab.c 3d72c780d1ea08906a198e4f033921a658a54590e3ed72c544995d84f3f9464a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d +F src/wal.c 0f34033977b2275793c4330b2ebc3fa180a1baee06591cbc8f6e0d7aaa37988d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 9a44063e60d8f42dd9dc8147b8e8dcfc315bbd13e25c395211292c36d828c869 @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3c04d21e6c632feb3bea8d1fa76bedcbfe254b0dc59865633d158a3f1bddefba -R 14e47e4bb1ab8f4db4305d42d7534790 +P a1c090e08139f99d30aa89db0756dc59fe8990ce15b3db4d4b726cc6acdab46f +R fed5903c05b22f31489b7dc87a54bece U drh -Z 5c670065e3a48de39ba2455ceb117563 +Z d121505cf61aa3dde02d007851da9f4e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9df82a5fbc..3778ae1c49 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1c090e08139f99d30aa89db0756dc59fe8990ce15b3db4d4b726cc6acdab46f \ No newline at end of file +1a8c2e54375ee2cf73773b798fed0ae07b42f5e068fddc513c093de5c1f46615 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index fdc4ac39b6..e7c1c053bd 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3138,20 +3138,17 @@ int sqlite3WalFindFrame( u32 *piRead /* OUT: Frame number (or zero) */ ){ u32 iRead = 0; /* If !=0, WAL frame to return data from */ - u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ + u32 iLast; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ int iMinHash; /* This routine is only be called from within a read transaction. */ assert( pWal->readLock>=0 || pWal->lockError ); - /* If the "last page" field of the wal-index header snapshot is 0, then - ** no data will be read from the wal under any circumstances. Return early - ** in this case as an optimization. Likewise, if pWal->readLock==0, - ** then the WAL is ignored by the reader so return early, as if the - ** WAL were empty. + /* if pWal->readLock==0, then the WAL is ignored by the reader + ** so return early, as if the WAL were empty. */ - if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ + if( pWal->readLock==0 && pWal->bShmUnreliable==0 ){ *piRead = 0; return SQLITE_OK; } @@ -3182,6 +3179,7 @@ int sqlite3WalFindFrame( ** table after the current read-transaction had started. */ iMinHash = walFramePage(pWal->minFrame); + iLast = pWal->hdr.mxFrame; for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){ WalHashLoc sLoc; /* Hash table location */ int iKey; /* Hash slot index */ From 48cae1320b49473787284ace2c543d1fd9332787 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 5 Jul 2022 10:40:30 +0000 Subject: [PATCH 028/151] Fix an assert() in btree to be more precise, as the previous form of that asser might fail due to recent performance optimizations. FossilOrigin-Name: 4774938134d0105423720bdc1b4e82164a1e28d993c5cd81c1b03f1f0878427e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 41d014f90b..ba0bd535d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Size\sreduction\sand\sperformance\soptimization\sin\ssqlite3WalFindFrame(). -D 2022-07-04T15:14:25.563 +C Fix\san\sassert()\sin\sbtree\sto\sbe\smore\sprecise,\sas\sthe\sprevious\sform\sof\sthat\nasser\smight\sfail\sdue\sto\srecent\sperformance\soptimizations. +D 2022-07-05T10:40:30.126 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c d4037518cecba5f9f69be57800c71eb798848b82cb6940511fdf8c7af726a70a +F src/btree.c 8923fefe168ffe5b943e053ce4869069e93545af79bb07028d9b08a70957aea4 F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c 23f874642825d7eaaeeb7a3281b2b1a75e1d4c4dd9ae4dceddcd908266634214 @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a1c090e08139f99d30aa89db0756dc59fe8990ce15b3db4d4b726cc6acdab46f -R fed5903c05b22f31489b7dc87a54bece +P 1a8c2e54375ee2cf73773b798fed0ae07b42f5e068fddc513c093de5c1f46615 +R 2757a3df013fcba9ceda084162d54bbf U drh -Z d121505cf61aa3dde02d007851da9f4e +Z 1c5dbf09a49cb9834da7c27397ce6fb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3778ae1c49..c0e86076d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a8c2e54375ee2cf73773b798fed0ae07b42f5e068fddc513c093de5c1f46615 \ No newline at end of file +4774938134d0105423720bdc1b4e82164a1e28d993c5cd81c1b03f1f0878427e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a9a4be99fc..c80a81ccc7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2309,9 +2309,7 @@ getAndInitPage_error1: pCur->pPage = pCur->apPage[pCur->iPage]; } testcase( pgno==0 ); - assert( pgno!=0 || rc==SQLITE_CORRUPT - || rc==SQLITE_IOERR_NOMEM - || rc==SQLITE_NOMEM ); + assert( pgno!=0 || rc!=SQLITE_OK ); return rc; } From c578e4f0eb6a99b52c710d21afbe722c746a821a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 5 Jul 2022 17:49:04 +0000 Subject: [PATCH 029/151] Fix a problem in the csv extension that was triggered when the very first field in the csv file is zero bytes in size. FossilOrigin-Name: b12ddabf07b5e06bcee8dda3f990af3a131ab52b8fa969ba061b26d38458f31d --- ext/misc/csv.c | 7 +++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/csv01.test | 22 ++++++++++++++++++++++ 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/ext/misc/csv.c b/ext/misc/csv.c index b51fbad30e..915c1cef8c 100644 --- a/ext/misc/csv.c +++ b/ext/misc/csv.c @@ -280,6 +280,7 @@ static char *csv_read_one_field(CsvReader *p){ } p->cTerm = (char)c; } + assert( p->z==0 || p->nnAlloc ); if( p->z ) p->z[p->n] = 0; p->bNotFirst = 1; return p->z; @@ -811,6 +812,12 @@ static int csvtabFilter( CsvCursor *pCur = (CsvCursor*)pVtabCursor; CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab; pCur->iRowid = 0; + + /* Ensure the field buffer is always allocated. Otherwise, if the + ** first field is zero bytes in size, this may be mistaken for an OOM + ** error in csvtabNext(). */ + if( csv_append(&pCur->rdr, 0) ) return SQLITE_NOMEM; + if( pCur->rdr.in==0 ){ assert( pCur->rdr.zIn==pTab->zData ); assert( pTab->iStart>=0 ); diff --git a/manifest b/manifest index ba0bd535d6..b55c72b924 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\sin\sbtree\sto\sbe\smore\sprecise,\sas\sthe\sprevious\sform\sof\sthat\nasser\smight\sfail\sdue\sto\srecent\sperformance\soptimizations. -D 2022-07-05T10:40:30.126 +C Fix\sa\sproblem\sin\sthe\scsv\sextension\sthat\swas\striggered\swhen\sthe\svery\sfirst\sfield\sin\sthe\scsv\sfile\sis\szero\sbytes\sin\ssize. +D 2022-07-05T17:49:04.076 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -314,7 +314,7 @@ F ext/misc/cksumvfs.c b42ef52eaaa510d54ec320c87bea149e934a3b06cd232be2093562bf66 F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c85c243 F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 -F ext/misc/csv.c d14709096280dc0e20c533f184568952bf4b8022ea80afc4aa9fec5ab3637bb3 +F ext/misc/csv.c ed30e56908a42499e3f167bbc0a2366a2226d020e2f6ef84d0cf8c191686ebd6 F ext/misc/dbdata.c e316fba936571584e55abd5b974a32a191727a6b746053a0c9d439bd2cf93940 F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 F ext/misc/decimal.c 09f967dcf4a1ee35a76309829308ec278d3648168733f4a1147820e11ebefd12 @@ -853,7 +853,7 @@ F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2 F test/createtab.test 85cdfdae5c3de331cd888d6c66e1aba575b47c2e3c3cc4a1d6f54140699f5165 F test/cse.test 00b3aea44b16828833c94fbe92475fd6977583fcb064ae0bc590986812b38d0c -F test/csv01.test c9c3af0d58c34e9ac970c5875a77939edb958762c8aafb95409e19a3f088b6cd +F test/csv01.test f3c76474e8e4aed7a0008065e4da4ab9fb7967712eb2037c13b478def0227c72 F test/ctime.test 340f362f41f92972bbd71f44e10569a5cc694062b692231bd08aa6fe6c1c4773 F test/cursorhint.test 0175e4404181ace3ceca8b114eb0a98eae600d565aa4e2705abbe6614c7fe201 F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a8c2e54375ee2cf73773b798fed0ae07b42f5e068fddc513c093de5c1f46615 -R 2757a3df013fcba9ceda084162d54bbf -U drh -Z 1c5dbf09a49cb9834da7c27397ce6fb7 +P 4774938134d0105423720bdc1b4e82164a1e28d993c5cd81c1b03f1f0878427e +R dfc0f75d3f385da37032208862505139 +U dan +Z c4ecb4b4b9fc154ab5d210ccd69309f6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c0e86076d8..8e715685d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4774938134d0105423720bdc1b4e82164a1e28d993c5cd81c1b03f1f0878427e \ No newline at end of file +b12ddabf07b5e06bcee8dda3f990af3a131ab52b8fa969ba061b26d38458f31d \ No newline at end of file diff --git a/test/csv01.test b/test/csv01.test index 9ba04206b8..288260661a 100644 --- a/test/csv01.test +++ b/test/csv01.test @@ -237,4 +237,26 @@ do_execsql_test 5.4 { SELECT *, '|' FROM t5_1; } {1 2 3 4 | one two three four | 5 6 7 8 |} +#------------------------------------------------------------------------- + +proc randomtext {n} { + string range [db one {SELECT hex(randomblob($n))}] 1 $n +} + +for {set ii 0} {$ii < 200} {incr ii} { + reset_db + load_static_extension db csv + set fd [open csv.data w] + puts $fd "a,b" + puts $fd "[randomtext $ii],abcd" + close $fd + do_execsql_test 6.$ii.1 { + CREATE VIRTUAL TABLE abc USING csv(filename='csv.data', header=true); + } + + do_execsql_test 6.$ii.2 { + SELECT count(*) FROM abc + } 1 +} + finish_test From f53487a42da5d564f7f5832f4708d8d7a6ccd73b Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 5 Jul 2022 19:53:59 +0000 Subject: [PATCH 030/151] Fix for builds with both SQLITE_DEBUG and SQLITE_OMIT_WINDOWFUNC defined. FossilOrigin-Name: 77916947ce3f0828e1c50bcb8a6438c951ab9e74d83ec4324e82e90e100b9a98 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b55c72b924..3aa8127796 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\scsv\sextension\sthat\swas\striggered\swhen\sthe\svery\sfirst\sfield\sin\sthe\scsv\sfile\sis\szero\sbytes\sin\ssize. -D 2022-07-05T17:49:04.076 +C Fix\sfor\sbuilds\swith\sboth\sSQLITE_DEBUG\sand\sSQLITE_OMIT_WINDOWFUNC\sdefined. +D 2022-07-05T19:53:59.705 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -534,7 +534,7 @@ F src/insert.c 173845e5a6bac96ae937409e4f876b631f26b31dabb9df8fd0eb3b130b2bb3a7 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b -F src/main.c e946f1d9e30bf49cf399ddf860cfd2319e3a97bbaabfb8e87d6ab985a20d54ad +F src/main.c 62100512f2f86e8af2e17cc41644a358868d5b26c691118b6ef31fea066d49d3 F src/malloc.c a9127efdcef92d6934c6339ea9813075b90edc0ce2e5c723556381a3828fb720 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -1978,8 +1978,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4774938134d0105423720bdc1b4e82164a1e28d993c5cd81c1b03f1f0878427e -R dfc0f75d3f385da37032208862505139 +P b12ddabf07b5e06bcee8dda3f990af3a131ab52b8fa969ba061b26d38458f31d +R 181929b813c3d31abfcc4b23a70a4427 U dan -Z c4ecb4b4b9fc154ab5d210ccd69309f6 +Z 61209a94126e8e0bbb9b2674f8f16b4a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8e715685d6..d7cce48213 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b12ddabf07b5e06bcee8dda3f990af3a131ab52b8fa969ba061b26d38458f31d \ No newline at end of file +77916947ce3f0828e1c50bcb8a6438c951ab9e74d83ec4324e82e90e100b9a98 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 3ae1f4fe30..6865121dd9 100644 --- a/src/main.c +++ b/src/main.c @@ -4112,8 +4112,10 @@ int sqlite3_test_control(int op, ...){ sqlite3ShowTriggerStepList(0); sqlite3ShowTrigger(0); sqlite3ShowTriggerList(0); +#ifndef SQLITE_OMIT_WINDOWFUNC sqlite3ShowWindow(0); sqlite3ShowWinFunc(0); +#endif } #endif break; From e36281fc31a171aea4406baa573a4a04cd89ab9d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Jul 2022 13:59:45 +0000 Subject: [PATCH 031/151] Avoid dropping error codes in the xBegin() method of virtual table sqlite_dbpage. FossilOrigin-Name: 570e2bce595d3a48977dca0bb573dd3fe745a0d13b7122b3ad8ab4745c0700d0 --- manifest | 13 ++++++----- manifest.uuid | 2 +- src/dbpage.c | 7 +++--- test/dbpagefault.test | 52 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 test/dbpagefault.test diff --git a/manifest b/manifest index 3aa8127796..3e17c90a1b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sfor\sbuilds\swith\sboth\sSQLITE_DEBUG\sand\sSQLITE_OMIT_WINDOWFUNC\sdefined. -D 2022-07-05T19:53:59.705 +C Avoid\sdropping\serror\scodes\sin\sthe\sxBegin()\smethod\sof\svirtual\stable\ssqlite_dbpage. +D 2022-07-06T13:59:45.908 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -518,7 +518,7 @@ F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad F src/date.c 15082566229d4b1e5f24fdb490bf9bcc68824b911d70e3573ef075a1b9e2d26f -F src/dbpage.c 90661a87e1db8bfbc8d2ebbdcd3749651ddb287c555c07a28fb17c7c591ffb68 +F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c a8e844af211a48b13b5b358be77a12c860c6a557c21990ad51a548e2536500ce F src/expr.c 4907afcb86d72b5525d8767515ce425ec53c7a2d3664441b46cef5b376ee0cba @@ -867,6 +867,7 @@ F test/dbfuzz001.test 55e1a3504f8dea84155e09912fe3b1c3ad77e0b1a938ec42ca03b8e51b F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee F test/dbfuzz2.c 4b3c12de4d98b1b2d908ab03d217d4619e47c8b23d5e67f8a6f2b1bdee7cae23 F test/dbpage.test fce29035c7566fd7835ec0f19422cb4b9c6944ce0e1b936ff8452443f92e887d +F test/dbpagefault.test e917cd250018b836e8d7d1a659c63bcd25737a8f31f15925b891b8f92247fdf9 F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed42759 F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef F test/decimal.test fcf403fd5585f47342234e153c4a4338cd737b8e0884ac66fc484df47dbcf1a7 @@ -1978,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b12ddabf07b5e06bcee8dda3f990af3a131ab52b8fa969ba061b26d38458f31d -R 181929b813c3d31abfcc4b23a70a4427 +P 77916947ce3f0828e1c50bcb8a6438c951ab9e74d83ec4324e82e90e100b9a98 +R e13916bae86c24145a230aedfd5b02fd U dan -Z 61209a94126e8e0bbb9b2674f8f16b4a +Z 3a52626dc6b0f09ba85ace5a67f4f7d5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d7cce48213..0ee36647a2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77916947ce3f0828e1c50bcb8a6438c951ab9e74d83ec4324e82e90e100b9a98 \ No newline at end of file +570e2bce595d3a48977dca0bb573dd3fe745a0d13b7122b3ad8ab4745c0700d0 \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index 003997b95f..9b565177c5 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -372,11 +372,12 @@ static int dbpageBegin(sqlite3_vtab *pVtab){ DbpageTable *pTab = (DbpageTable *)pVtab; sqlite3 *db = pTab->db; int i; - for(i=0; inDb; i++){ + int rc = SQLITE_OK; + for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( pBt ) sqlite3BtreeBeginTrans(pBt, 1, 0); + if( pBt ) rc = sqlite3BtreeBeginTrans(pBt, 1, 0); } - return SQLITE_OK; + return rc; } diff --git a/test/dbpagefault.test b/test/dbpagefault.test new file mode 100644 index 0000000000..550a567127 --- /dev/null +++ b/test/dbpagefault.test @@ -0,0 +1,52 @@ +# 2022 July 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl + +if {[permutation] == "inmemory_journal"} { + finish_test + return +} + +set testprefix dbpagefault + + +faultsim_save_and_close +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen + execsql { ATTACH 'test.db2' AS aux; } +} -body { + execsql { + CREATE VIRTUAL TABLE t1 USING sqlite_dbpage(); + } +} -test { + execsql { PRAGMA journal_mode = off } + faultsim_test_result {0 {}} +} + +do_faultsim_test 2 -prep { + sqlite3 db "xyz.db" -vfs memdb + execsql { ATTACH 'test.db2' AS aux; } +} -body { + execsql { + CREATE VIRTUAL TABLE t1 USING sqlite_dbpage(); + INSERT INTO t1 DEFAULT VALUES; + } +} -test { + execsql { PRAGMA journal_mode = off } + faultsim_test_result {1 {no such schema}} {1 {SQL logic error}} +} + +finish_test From e7ebe0aa81902f9f9e9b8a338b33509289e43c74 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Jul 2022 15:44:57 +0000 Subject: [PATCH 032/151] Avoid ignoring the last line of a csv file if the final field is empty and there is no trailing CFLS. Also have the csv extension treat the last line of a file in the same way as any other line if it is short fields. FossilOrigin-Name: 587795d47fcaf5142526fabbcc4d5a632f561f258414c2846e8932a49b5b2e6b --- ext/misc/csv.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- test/csv01.test | 19 ++++++++++++++++++- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/ext/misc/csv.c b/ext/misc/csv.c index 915c1cef8c..870a0cf60e 100644 --- a/ext/misc/csv.c +++ b/ext/misc/csv.c @@ -751,7 +751,7 @@ static int csvtabNext(sqlite3_vtab_cursor *cur){ i++; } }while( pCur->rdr.cTerm==',' ); - if( z==0 || (pCur->rdr.cTerm==EOF && inCol) ){ + if( z==0 && i==0 ){ pCur->iRowid = -1; }else{ pCur->iRowid++; diff --git a/manifest b/manifest index 3e17c90a1b..5199cc80d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sdropping\serror\scodes\sin\sthe\sxBegin()\smethod\sof\svirtual\stable\ssqlite_dbpage. -D 2022-07-06T13:59:45.908 +C Avoid\signoring\sthe\slast\sline\sof\sa\scsv\sfile\sif\sthe\sfinal\sfield\sis\sempty\sand\nthere\sis\sno\strailing\sCFLS.\sAlso\shave\sthe\scsv\sextension\streat\sthe\slast\sline\sof\sa\nfile\sin\sthe\ssame\sway\sas\sany\sother\sline\sif\sit\sis\sshort\sfields. +D 2022-07-06T15:44:57.999 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -314,7 +314,7 @@ F ext/misc/cksumvfs.c b42ef52eaaa510d54ec320c87bea149e934a3b06cd232be2093562bf66 F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c85c243 F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 -F ext/misc/csv.c ed30e56908a42499e3f167bbc0a2366a2226d020e2f6ef84d0cf8c191686ebd6 +F ext/misc/csv.c ca8d6dafc5469639de81937cb66ae2e6b358542aba94c4f791910d355a8e7f73 F ext/misc/dbdata.c e316fba936571584e55abd5b974a32a191727a6b746053a0c9d439bd2cf93940 F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 F ext/misc/decimal.c 09f967dcf4a1ee35a76309829308ec278d3648168733f4a1147820e11ebefd12 @@ -853,7 +853,7 @@ F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2 F test/createtab.test 85cdfdae5c3de331cd888d6c66e1aba575b47c2e3c3cc4a1d6f54140699f5165 F test/cse.test 00b3aea44b16828833c94fbe92475fd6977583fcb064ae0bc590986812b38d0c -F test/csv01.test f3c76474e8e4aed7a0008065e4da4ab9fb7967712eb2037c13b478def0227c72 +F test/csv01.test 2ab5514005fd308995c8910bc313e47f0368b94213b9d6c27f9a2da78796a091 F test/ctime.test 340f362f41f92972bbd71f44e10569a5cc694062b692231bd08aa6fe6c1c4773 F test/cursorhint.test 0175e4404181ace3ceca8b114eb0a98eae600d565aa4e2705abbe6614c7fe201 F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 77916947ce3f0828e1c50bcb8a6438c951ab9e74d83ec4324e82e90e100b9a98 -R e13916bae86c24145a230aedfd5b02fd +P 570e2bce595d3a48977dca0bb573dd3fe745a0d13b7122b3ad8ab4745c0700d0 +R de3e674aa770d8389e09eb000a6ba987 U dan -Z 3a52626dc6b0f09ba85ace5a67f4f7d5 +Z 390a28cf69c24072416f697a32ef7273 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0ee36647a2..548817a3cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -570e2bce595d3a48977dca0bb573dd3fe745a0d13b7122b3ad8ab4745c0700d0 \ No newline at end of file +587795d47fcaf5142526fabbcc4d5a632f561f258414c2846e8932a49b5b2e6b \ No newline at end of file diff --git a/test/csv01.test b/test/csv01.test index 288260661a..ecb1a968de 100644 --- a/test/csv01.test +++ b/test/csv01.test @@ -253,10 +253,27 @@ for {set ii 0} {$ii < 200} {incr ii} { do_execsql_test 6.$ii.1 { CREATE VIRTUAL TABLE abc USING csv(filename='csv.data', header=true); } - do_execsql_test 6.$ii.2 { SELECT count(*) FROM abc } 1 } +for {set ii 0} {$ii < 20} {incr ii} { + reset_db + load_static_extension db csv + set T [randomtext $ii] + set fd [open csv.data w] + puts $fd "a,b" + puts -nonewline $fd "abcd,$T" + close $fd + do_execsql_test 7.$ii.1 { + CREATE VIRTUAL TABLE abc USING csv(filename='csv.data', header=true); + } + breakpoint + do_execsql_test 7.$ii.2 { + SELECT * FROM abc + } [list abcd $T] +} + + finish_test From 07fae32db07cf4f8e2d7e8e143663fb84a8c8116 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 6 Jul 2022 23:50:01 +0000 Subject: [PATCH 033/151] Fix harmless compiler warnings seen with MSVC. FossilOrigin-Name: 61e2094afbbcbd5fdf5c3ec06b96134fafb7b854dc9bfa7d0619bed6d35efbe4 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/resolve.c | 2 +- src/select.c | 4 ++-- src/shell.c.in | 2 +- src/treeview.c | 6 +++--- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 5199cc80d9..032b39b8a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\signoring\sthe\slast\sline\sof\sa\scsv\sfile\sif\sthe\sfinal\sfield\sis\sempty\sand\nthere\sis\sno\strailing\sCFLS.\sAlso\shave\sthe\scsv\sextension\streat\sthe\slast\sline\sof\sa\nfile\sin\sthe\ssame\sway\sas\sany\sother\sline\sif\sit\sis\sshort\sfields. -D 2022-07-06T15:44:57.999 +C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC. +D 2022-07-06T23:50:01.037 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -568,10 +568,10 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c 6166a30417b05c5b2f82e1f183f75faa2926ad60531c0b688a57dbc951441a20 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c -F src/resolve.c 1655e44c77c51ebbe82924287528a78bd4a4aaaf34189dbae28d19ccf2ca615c +F src/resolve.c 84a8443e3723e908730d754f54df4e1dacc1eb7073c0bd79c9d5efe977a9f5b9 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c fee45b7866181d86cb4e666adf148061166fbb4eb4df72ebc756901d48ba4025 -F src/shell.c.in 08e59f1cb9d9b1180aba52861aaada0c95f6ddd210488719684e160a0724c806 +F src/select.c 3c6558bb0678a4d1d086c30afbd933c1160356aff5285825f6cf0de66b81f16c +F src/shell.c.in 2b85128ca8ea13fc2dc32f971d628d9f688a324a30f469619b817ce490764fcb F src/sqlite.h.in 01573eae96721f2a8ee2a9e3b7140ceeba2e9c44350911890b89b8ff0dcf6781 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d @@ -635,7 +635,7 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c a38f52058b517929e264094abd0b5fd1e8e145a1aa43bc6f6a72ae5218f96c98 -F src/treeview.c 0bd550f5974565eba541db339906be7a3f9e91beafb879cb40705382887f08e1 +F src/treeview.c 4d8eda242386ca9f47276c5b62cb41922587d4e27002acfb82c58e85dac4d93a F src/trigger.c 61bea163b1fa3039bc572ed8312461b978e5c527e5301f302b078f4c1ccdec6a F src/update.c c52a7991bece0453d22c77c08469512ee2f1391c12503fd347d1c939220c5877 F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 570e2bce595d3a48977dca0bb573dd3fe745a0d13b7122b3ad8ab4745c0700d0 -R de3e674aa770d8389e09eb000a6ba987 -U dan -Z 390a28cf69c24072416f697a32ef7273 +P 587795d47fcaf5142526fabbcc4d5a632f561f258414c2846e8932a49b5b2e6b +R 2eb751598396b97b93bf551986a07087 +U mistachkin +Z 4d05d451769cfa8ea2dbf377223e0684 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 548817a3cb..a0ba389f3a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -587795d47fcaf5142526fabbcc4d5a632f561f258414c2846e8932a49b5b2e6b \ No newline at end of file +61e2094afbbcbd5fdf5c3ec06b96134fafb7b854dc9bfa7d0619bed6d35efbe4 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 6a36c8cf47..99e30d4c83 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -314,7 +314,7 @@ static int lookupName( pTab = pItem->pTab; assert( pTab!=0 && pTab->zName!=0 ); assert( pTab->nCol>0 || pParse->nErr ); - assert( pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); if( pItem->fg.isNestedFrom ){ /* In this case, pItem is a subquery that has been formed from a ** parenthesized subset of the FROM clause terms. Example: diff --git a/src/select.c b/src/select.c index fcd8247d22..76b20d694a 100644 --- a/src/select.c +++ b/src/select.c @@ -324,7 +324,7 @@ int sqlite3ColumnIndex(Table *pTab, const char *zCol){ */ void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){ assert( pItem!=0 ); - assert( pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); if( pItem->fg.isNestedFrom ){ ExprList *pResults; assert( pItem->pSelect!=0 ); @@ -5895,7 +5895,7 @@ static int selectExpander(Walker *pWalker, Select *p){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - assert( pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) ); + assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) ); if( pFrom->fg.isNestedFrom ){ assert( pFrom->pSelect!=0 ); pNestedFrom = pFrom->pSelect->pEList; diff --git a/src/shell.c.in b/src/shell.c.in index 1d86e1eed6..4e921cb6e4 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11990,7 +11990,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char **argv; #endif #ifdef SQLITE_DEBUG - sqlite3_uint64 mem_main_enter = sqlite3_memory_used(); + sqlite3_int64 mem_main_enter = sqlite3_memory_used(); #endif char *zErrMsg = 0; #ifdef SQLITE_SHELL_WASM_MODE diff --git a/src/treeview.c b/src/treeview.c index 366c4afd18..36bfc29d13 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -100,8 +100,8 @@ void sqlite3TreeViewColumnList( sqlite3TreeViewLine(pView, "COLUMNS"); for(i=0; ipTab; sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); } - assert( pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); sqlite3TreeViewSelect(pView, pItem->pSelect, (--n)>0); } if( pItem->fg.isTabFunc ){ From dce232a29813bcd8ab175529f1b98172cf6b7506 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 7 Jul 2022 20:11:35 +0000 Subject: [PATCH 034/151] Size reduction and performance increase in the freeSpace() routine of btree.c. FossilOrigin-Name: 7d7aed053f600659c63d8bac6d5da77879936c3fc57bfd058a4943b3bc530575 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 032b39b8a9..bec949f30c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC. -D 2022-07-06T23:50:01.037 +C Size\sreduction\sand\sperformance\sincrease\sin\sthe\sfreeSpace()\sroutine\sof\sbtree.c. +D 2022-07-07T20:11:35.233 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 8923fefe168ffe5b943e053ce4869069e93545af79bb07028d9b08a70957aea4 +F src/btree.c 4cba0082bd53caf0897b743ed28b0e4c588c3c79a785042fab63cbdd41184807 F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c 23f874642825d7eaaeeb7a3281b2b1a75e1d4c4dd9ae4dceddcd908266634214 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 587795d47fcaf5142526fabbcc4d5a632f561f258414c2846e8932a49b5b2e6b -R 2eb751598396b97b93bf551986a07087 -U mistachkin -Z 4d05d451769cfa8ea2dbf377223e0684 +P 61e2094afbbcbd5fdf5c3ec06b96134fafb7b854dc9bfa7d0619bed6d35efbe4 +R 6b7426ab3817b569b11bfb68e4612c05 +U drh +Z e6c220e352c0a10417bd5db52eaf6a42 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a0ba389f3a..7dad21dffd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61e2094afbbcbd5fdf5c3ec06b96134fafb7b854dc9bfa7d0619bed6d35efbe4 \ No newline at end of file +7d7aed053f600659c63d8bac6d5da77879936c3fc57bfd058a4943b3bc530575 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c80a81ccc7..97a5c9e527 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1827,7 +1827,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ }else{ while( (iFreeBlk = get2byte(&data[iPtr])) Date: Thu, 7 Jul 2022 20:29:49 +0000 Subject: [PATCH 035/151] Size reduction and performance increase in the pageFindSlot() routine of btree.c. FossilOrigin-Name: 5d247e38560b97975568e8a48324acaca1002ec2ef7fa3efc9c8aa7d83412aad --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bec949f30c..06040bca47 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Size\sreduction\sand\sperformance\sincrease\sin\sthe\sfreeSpace()\sroutine\sof\sbtree.c. -D 2022-07-07T20:11:35.233 +C Size\sreduction\sand\sperformance\sincrease\sin\sthe\spageFindSlot()\sroutine\sof\nbtree.c. +D 2022-07-07T20:29:49.982 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 4cba0082bd53caf0897b743ed28b0e4c588c3c79a785042fab63cbdd41184807 +F src/btree.c 6599d972b674cbbcb76e9a9db0e206d3bf9b60186464c08ef82ab6fda0622d44 F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c 23f874642825d7eaaeeb7a3281b2b1a75e1d4c4dd9ae4dceddcd908266634214 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 61e2094afbbcbd5fdf5c3ec06b96134fafb7b854dc9bfa7d0619bed6d35efbe4 -R 6b7426ab3817b569b11bfb68e4612c05 +P 7d7aed053f600659c63d8bac6d5da77879936c3fc57bfd058a4943b3bc530575 +R 573750a14fc9838f7267df2add226c79 U drh -Z e6c220e352c0a10417bd5db52eaf6a42 +Z 6e9989a326ed7411def6db942d22ad53 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7dad21dffd..5f4865c9bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7d7aed053f600659c63d8bac6d5da77879936c3fc57bfd058a4943b3bc530575 \ No newline at end of file +5d247e38560b97975568e8a48324acaca1002ec2ef7fa3efc9c8aa7d83412aad \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 97a5c9e527..2d2a20cd52 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1673,9 +1673,9 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ iAddr = pc; pTmp = &aData[pc]; pc = get2byte(pTmp); - if( pc<=iAddr+size ){ + if( pc<=iAddr ){ if( pc ){ - /* The next slot in the chain is not past the end of the current slot */ + /* The next slot in the chain comes before the current slot */ *pRc = SQLITE_CORRUPT_PAGE(pPg); } return 0; From c59b7b1fb03ddef422143cacdc40fdbc531f8a74 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 7 Jul 2022 20:49:22 +0000 Subject: [PATCH 036/151] Upgrade the TEA build system in autoconf/tea/. To match tclconfig commit 20fe9e6f59 and Tcl Sample Extension be47fb0446. FossilOrigin-Name: 1531f7391890d7d3cd091c2d1284230f128e5282bf676967ebcb212210e51e71 --- autoconf/tea/Makefile.in | 183 +++--- autoconf/tea/configure.ac | 86 ++- autoconf/tea/pkgIndex.tcl.in | 15 +- autoconf/tea/tclconfig/tcl.m4 | 1151 +++++++++++++++------------------ manifest | 23 +- manifest.uuid | 2 +- 6 files changed, 709 insertions(+), 751 deletions(-) diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index 3e481dadfe..5264f89f3c 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -11,8 +11,6 @@ # # See the file "license.terms" for information on usage and redistribution # of this file, and for a DISCLAIMER OF ALL WARRANTIES. -# -# RCS: @(#) $Id: Makefile.in,v 1.59 2005/07/26 19:17:02 mdejong Exp $ #======================================================================== # Add additional lines to handle any additional AC_SUBST cases that @@ -60,6 +58,8 @@ PKG_HEADERS = @PKG_HEADERS@ #======================================================================== PKG_LIB_FILE = @PKG_LIB_FILE@ +PKG_LIB_FILE8 = @PKG_LIB_FILE8@ +PKG_LIB_FILE9 = @PKG_LIB_FILE9@ PKG_STUB_LIB_FILE = @PKG_STUB_LIB_FILE@ lib_BINARIES = $(PKG_LIB_FILE) @@ -73,10 +73,11 @@ exec_prefix = @exec_prefix@ bindir = @bindir@ libdir = @libdir@ +includedir = @includedir@ datarootdir = @datarootdir@ +runstatedir = @runstatedir@ datadir = @datadir@ mandir = @mandir@ -includedir = @includedir@ DESTDIR = @@ -85,24 +86,25 @@ pkgdatadir = $(datadir)/$(PKG_DIR) pkglibdir = $(libdir)/$(PKG_DIR) pkgincludedir = $(includedir)/$(PKG_DIR) -top_builddir = . +top_builddir = @abs_top_builddir@ -INSTALL = @INSTALL@ -INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_OPTIONS = +INSTALL = @INSTALL@ $(INSTALL_OPTIONS) +INSTALL_DATA_DIR = @INSTALL_DATA_DIR@ INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_LIBRARY = @INSTALL_LIBRARY@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ CC = @CC@ +CCLD = @CCLD@ CFLAGS_DEFAULT = @CFLAGS_DEFAULT@ CFLAGS_WARNING = @CFLAGS_WARNING@ -CLEANFILES = @CLEANFILES@ EXEEXT = @EXEEXT@ LDFLAGS_DEFAULT = @LDFLAGS_DEFAULT@ MAKE_LIB = @MAKE_LIB@ -MAKE_SHARED_LIB = @MAKE_SHARED_LIB@ -MAKE_STATIC_LIB = @MAKE_STATIC_LIB@ MAKE_STUB_LIB = @MAKE_STUB_LIB@ OBJEXT = @OBJEXT@ RANLIB = @RANLIB@ @@ -117,8 +119,6 @@ TCL_SRC_DIR = @TCL_SRC_DIR@ #TK_BIN_DIR = @TK_BIN_DIR@ #TK_SRC_DIR = @TK_SRC_DIR@ -# This is no longer necessary even for packages that use private Tcl headers -#TCL_TOP_DIR_NATIVE = @TCL_TOP_DIR_NATIVE@ # Not used, but retained for reference of what libs Tcl required #TCL_LIBS = @TCL_LIBS@ @@ -132,41 +132,52 @@ TCL_SRC_DIR = @TCL_SRC_DIR@ EXTRA_PATH = $(top_builddir):$(TCL_BIN_DIR) #EXTRA_PATH = $(top_builddir):$(TCL_BIN_DIR):$(TK_BIN_DIR) TCLLIBPATH = $(top_builddir) -TCLSH_ENV = TCL_LIBRARY=`@CYGPATH@ $(TCL_SRC_DIR)/library` \ - @LD_LIBRARY_PATH_VAR@="$(EXTRA_PATH):$(@LD_LIBRARY_PATH_VAR@)" \ +TCLSH_ENV = TCL_LIBRARY=`@CYGPATH@ $(TCL_SRC_DIR)/library` +PKG_ENV = @LD_LIBRARY_PATH_VAR@="$(EXTRA_PATH):$(@LD_LIBRARY_PATH_VAR@)" \ PATH="$(EXTRA_PATH):$(PATH)" \ TCLLIBPATH="$(TCLLIBPATH)" -# TK_LIBRARY=`@CYGPATH@ $(TK_SRC_DIR)/library` TCLSH_PROG = @TCLSH_PROG@ -TCLSH = $(TCLSH_ENV) $(TCLSH_PROG) +TCLSH = $(TCLSH_ENV) $(PKG_ENV) $(TCLSH_PROG) +#WISH_ENV = TK_LIBRARY=`@CYGPATH@ $(TK_SRC_DIR)/library` #WISH_PROG = @WISH_PROG@ -#WISH = $(TCLSH_ENV) $(WISH_PROG) - +#WISH = $(TCLSH_ENV) $(WISH_ENV) $(PKG_ENV) $(WISH_PROG) SHARED_BUILD = @SHARED_BUILD@ -INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ -I$(srcdir)/.. +INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ -I. -I$(srcdir)/.. #INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ @TK_INCLUDES@ @TK_XINCLUDES@ PKG_CFLAGS = @PKG_CFLAGS@ # TCL_DEFS is not strictly need here, but if you remove it, then you -# must make sure that configure.in checks for the necessary components +# must make sure that configure.ac checks for the necessary components # that your library may use. TCL_DEFS can actually be a problem if # you do not compile with a similar machine setup as the Tcl core was # compiled with. #DEFS = $(TCL_DEFS) @DEFS@ $(PKG_CFLAGS) DEFS = @DEFS@ $(PKG_CFLAGS) +# Move pkgIndex.tcl to 'BINARIES' var if it is generated in the Makefile CONFIG_CLEAN_FILES = Makefile pkgIndex.tcl +CLEANFILES = @CLEANFILES@ CPPFLAGS = @CPPFLAGS@ LIBS = @PKG_LIBS@ @LIBS@ AR = @AR@ CFLAGS = @CFLAGS@ -COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LDFLAGS = @LDFLAGS@ +LDFLAGS_DEFAULT = @LDFLAGS_DEFAULT@ +COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) \ + $(CFLAGS_DEFAULT) $(CFLAGS_WARNING) $(SHLIB_CFLAGS) $(CFLAGS) + +GDB = gdb +VALGRIND = valgrind +VALGRINDARGS = --tool=memcheck --num-callers=8 --leak-resolution=high \ + --leak-check=yes --show-reachable=yes -v + +.SUFFIXES: .c .$(OBJEXT) #======================================================================== # Start of user-definable TARGETS section @@ -174,7 +185,7 @@ COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(C #======================================================================== # TEA TARGETS. Please note that the "libraries:" target refers to platform -# independent files, and the "binaries:" target inclues executable programs and +# independent files, and the "binaries:" target includes executable programs and # platform-dependent libraries. Modify these targets so that they install # the various pieces of your package. The make and install rules # for the BINARIES that you specified above have already been done. @@ -193,7 +204,6 @@ binaries: $(BINARIES) libraries: - #======================================================================== # Your doc target should differentiate from doc builds (by the developer) # and doc installs (see install-doc), which just install the docs on the @@ -216,11 +226,11 @@ install-binaries: binaries install-lib-binaries install-bin-binaries #======================================================================== install-libraries: libraries - @mkdir -p $(DESTDIR)$(includedir) + @$(INSTALL_DATA_DIR) "$(DESTDIR)$(includedir)" @echo "Installing header files in $(DESTDIR)$(includedir)" @list='$(PKG_HEADERS)'; for i in $$list; do \ echo "Installing $(srcdir)/$$i" ; \ - $(INSTALL_DATA) $(srcdir)/$$i $(DESTDIR)$(includedir) ; \ + $(INSTALL_DATA) $(srcdir)/$$i "$(DESTDIR)$(includedir)" ; \ done; #======================================================================== @@ -229,12 +239,11 @@ install-libraries: libraries #======================================================================== install-doc: doc - @mkdir -p $(DESTDIR)$(mandir)/mann + @$(INSTALL_DATA_DIR) "$(DESTDIR)$(mandir)/mann" @echo "Installing documentation in $(DESTDIR)$(mandir)" @list='$(srcdir)/doc/*.n'; for i in $$list; do \ echo "Installing $$i"; \ - rm -f $(DESTDIR)$(mandir)/mann/`basename $$i`; \ - $(INSTALL_DATA) $$i $(DESTDIR)$(mandir)/mann ; \ + $(INSTALL_DATA) $$i "$(DESTDIR)$(mandir)/mann" ; \ done test: binaries libraries @@ -244,7 +253,21 @@ shell: binaries libraries @$(TCLSH) $(SCRIPT) gdb: - $(TCLSH_ENV) gdb $(TCLSH_PROG) $(SCRIPT) + $(TCLSH_ENV) $(PKG_ENV) $(GDB) $(TCLSH_PROG) $(SCRIPT) + +gdb-test: binaries libraries + $(TCLSH_ENV) $(PKG_ENV) $(GDB) \ + --args $(TCLSH_PROG) `@CYGPATH@ $(srcdir)/tests/all.tcl` \ + $(TESTFLAGS) -singleproc 1 \ + -load "package ifneeded $(PACKAGE_NAME) $(PACKAGE_VERSION) \ + [list load `@CYGPATH@ $(PKG_LIB_FILE)` [string totitle $(PACKAGE_NAME)]]" + +valgrind: binaries libraries + $(TCLSH_ENV) $(PKG_ENV) $(VALGRIND) $(VALGRINDARGS) $(TCLSH_PROG) \ + `@CYGPATH@ $(srcdir)/tests/all.tcl` $(TESTFLAGS) + +valgrindshell: binaries libraries + $(TCLSH_ENV) $(PKG_ENV) $(VALGRIND) $(VALGRINDARGS) $(TCLSH_PROG) $(SCRIPT) depend: @@ -283,49 +306,70 @@ $(PKG_STUB_LIB_FILE): $(PKG_STUB_OBJECTS) # As necessary, add $(srcdir):$(srcdir)/compat:.... #======================================================================== -VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win +VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win:$(srcdir)/macosx .c.@OBJEXT@: $(COMPILE) -c `@CYGPATH@ $<` -o $@ +tclsample.@OBJEXT@: sampleUuid.h + +$(srcdir)/manifest.uuid: + printf "git-" >$(srcdir)/manifest.uuid + (cd $(srcdir); git rev-parse HEAD >>$(srcdir)/manifest.uuid || \ + (printf "svn-r" >$(srcdir)/manifest.uuid ; \ + svn info --show-item last-changed-revision >>$(srcdir)/manifest.uuid) || \ + printf "unknown" >$(srcdir)/manifest.uuid) + +sampleUuid.h: $(srcdir)/manifest.uuid + echo "#define SAMPLE_VERSION_UUID \\" >$@ + cat $(srcdir)/manifest.uuid >>$@ + echo "" >>$@ + #======================================================================== # Distribution creation # You may need to tweak this target to make it work correctly. #======================================================================== #COMPRESS = tar cvf $(PKG_DIR).tar $(PKG_DIR); compress $(PKG_DIR).tar -COMPRESS = gtar zcvf $(PKG_DIR).tar.gz $(PKG_DIR) +COMPRESS = tar zcvf $(PKG_DIR).tar.gz $(PKG_DIR) DIST_ROOT = /tmp/dist DIST_DIR = $(DIST_ROOT)/$(PKG_DIR) +DIST_INSTALL_DATA = CPPROG='cp -p' $(INSTALL) -m 644 +DIST_INSTALL_SCRIPT = CPPROG='cp -p' $(INSTALL) -m 755 + dist-clean: rm -rf $(DIST_DIR) $(DIST_ROOT)/$(PKG_DIR).tar.* -dist: dist-clean - mkdir -p $(DIST_DIR) - cp -p $(srcdir)/README* $(srcdir)/license* \ - $(srcdir)/aclocal.m4 $(srcdir)/configure $(srcdir)/*.in \ - $(DIST_DIR)/ - chmod 664 $(DIST_DIR)/Makefile.in $(DIST_DIR)/aclocal.m4 - chmod 775 $(DIST_DIR)/configure $(DIST_DIR)/configure.in +dist: dist-clean $(srcdir)/manifest.uuid + $(INSTALL_DATA_DIR) $(DIST_DIR) - for i in $(srcdir)/*.[ch]; do \ - if [ -f $$i ]; then \ - cp -p $$i $(DIST_DIR)/ ; \ - fi; \ - done; + # TEA files + $(DIST_INSTALL_DATA) $(srcdir)/Makefile.in \ + $(srcdir)/aclocal.m4 $(srcdir)/configure.ac \ + $(DIST_DIR)/ + $(DIST_INSTALL_SCRIPT) $(srcdir)/configure $(DIST_DIR)/ - mkdir $(DIST_DIR)/tclconfig - cp $(srcdir)/tclconfig/install-sh $(srcdir)/tclconfig/tcl.m4 \ - $(DIST_DIR)/tclconfig/ - chmod 664 $(DIST_DIR)/tclconfig/tcl.m4 - chmod +x $(DIST_DIR)/tclconfig/install-sh + $(INSTALL_DATA_DIR) $(DIST_DIR)/tclconfig + $(DIST_INSTALL_DATA) $(srcdir)/tclconfig/README.txt \ + $(srcdir)/manifest.uuid \ + $(srcdir)/tclconfig/tcl.m4 $(srcdir)/tclconfig/install-sh \ + $(DIST_DIR)/tclconfig/ - list='demos doc generic library mac tests unix win'; \ + # Extension files + $(DIST_INSTALL_DATA) \ + $(srcdir)/ChangeLog \ + $(srcdir)/README.sha \ + $(srcdir)/license.terms \ + $(srcdir)/README \ + $(srcdir)/pkgIndex.tcl.in \ + $(DIST_DIR)/ + + list='demos doc generic library macosx tests unix win'; \ for p in $$list; do \ if test -d $(srcdir)/$$p ; then \ - mkdir $(DIST_DIR)/$$p; \ - cp -p $(srcdir)/$$p/*.* $(DIST_DIR)/$$p/; \ + $(INSTALL_DATA_DIR) $(DIST_DIR)/$$p; \ + $(DIST_INSTALL_DATA) $(srcdir)/$$p/* $(DIST_DIR)/$$p/; \ fi; \ done @@ -337,10 +381,10 @@ dist: dist-clean #======================================================================== # Don't modify the file to clean here. Instead, set the "CLEANFILES" -# variable in configure.in +# variable in configure.ac #======================================================================== -clean: +clean: -test -z "$(BINARIES)" || rm -f $(BINARIES) -rm -f *.$(OBJEXT) core *.core -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) @@ -348,7 +392,7 @@ clean: distclean: clean -rm -f *.tab.c -rm -f $(CONFIG_CLEAN_FILES) - -rm -f config.h config.cache config.log config.status + -rm -f config.cache config.log config.status #======================================================================== # Install binary object libraries. On Windows this includes both .dll and @@ -362,25 +406,17 @@ distclean: clean #======================================================================== install-lib-binaries: binaries - @mkdir -p $(DESTDIR)$(pkglibdir) + @$(INSTALL_DATA_DIR) "$(DESTDIR)$(pkglibdir)" @list='$(lib_BINARIES)'; for p in $$list; do \ if test -f $$p; then \ - echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(pkglibdir)/$$p"; \ - $(INSTALL_PROGRAM) $$p $(DESTDIR)$(pkglibdir)/$$p; \ - stub=`echo $$p|sed -e "s/.*\(stub\).*/\1/"`; \ - if test "x$$stub" = "xstub"; then \ - echo " $(RANLIB_STUB) $(DESTDIR)$(pkglibdir)/$$p"; \ - $(RANLIB_STUB) $(DESTDIR)$(pkglibdir)/$$p; \ - else \ - echo " $(RANLIB) $(DESTDIR)$(pkglibdir)/$$p"; \ - $(RANLIB) $(DESTDIR)$(pkglibdir)/$$p; \ - fi; \ + echo " $(INSTALL_LIBRARY) $$p $(DESTDIR)$(pkglibdir)/$$p"; \ + $(INSTALL_LIBRARY) $$p "$(DESTDIR)$(pkglibdir)/$$p"; \ ext=`echo $$p|sed -e "s/.*\.//"`; \ if test "x$$ext" = "xdll"; then \ lib=`basename $$p|sed -e 's/.[^.]*$$//'`.lib; \ if test -f $$lib; then \ echo " $(INSTALL_DATA) $$lib $(DESTDIR)$(pkglibdir)/$$lib"; \ - $(INSTALL_DATA) $$lib $(DESTDIR)$(pkglibdir)/$$lib; \ + $(INSTALL_DATA) $$lib "$(DESTDIR)$(pkglibdir)/$$lib"; \ fi; \ fi; \ fi; \ @@ -389,12 +425,12 @@ install-lib-binaries: binaries if test -f $(srcdir)/$$p; then \ destp=`basename $$p`; \ echo " Install $$destp $(DESTDIR)$(pkglibdir)/$$destp"; \ - $(INSTALL_DATA) $(srcdir)/$$p $(DESTDIR)$(pkglibdir)/$$destp; \ + $(INSTALL_DATA) $(srcdir)/$$p "$(DESTDIR)$(pkglibdir)/$$destp"; \ fi; \ done @if test "x$(SHARED_BUILD)" = "x1"; then \ echo " Install pkgIndex.tcl $(DESTDIR)$(pkglibdir)"; \ - $(INSTALL_DATA) pkgIndex.tcl $(DESTDIR)$(pkglibdir); \ + $(INSTALL_DATA) pkgIndex.tcl "$(DESTDIR)$(pkglibdir)"; \ fi #======================================================================== @@ -407,33 +443,32 @@ install-lib-binaries: binaries #======================================================================== install-bin-binaries: binaries - @mkdir -p $(DESTDIR)$(bindir) + @$(INSTALL_DATA_DIR) "$(DESTDIR)$(bindir)" @list='$(bin_BINARIES)'; for p in $$list; do \ if test -f $$p; then \ echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/$$p"; \ - $(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/$$p; \ + $(INSTALL_PROGRAM) $$p "$(DESTDIR)$(bindir)/$$p"; \ fi; \ done -.SUFFIXES: .c .$(OBJEXT) - Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status cd $(top_builddir) \ && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status uninstall-binaries: list='$(lib_BINARIES)'; for p in $$list; do \ - rm -f $(DESTDIR)$(pkglibdir)/$$p; \ + rm -f "$(DESTDIR)$(pkglibdir)/$$p"; \ done list='$(PKG_TCL_SOURCES)'; for p in $$list; do \ p=`basename $$p`; \ - rm -f $(DESTDIR)$(pkglibdir)/$$p; \ + rm -f "$(DESTDIR)$(pkglibdir)/$$p"; \ done list='$(bin_BINARIES)'; for p in $$list; do \ - rm -f $(DESTDIR)$(bindir)/$$p; \ + rm -f "$(DESTDIR)$(bindir)/$$p"; \ done .PHONY: all binaries clean depend distclean doc install libraries test +.PHONY: gdb gdb-test valgrind valgrindshell # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. diff --git a/autoconf/tea/configure.ac b/autoconf/tea/configure.ac index dae94f512e..90f418f3e4 100644 --- a/autoconf/tea/configure.ac +++ b/autoconf/tea/configure.ac @@ -2,11 +2,9 @@ dnl This file is an input file used by the GNU "autoconf" program to dnl generate the file "configure", which is run during Tcl installation dnl to configure the system for the local environment. -# -# RCS: @(#) $Id: configure.in,v 1.43 2005/07/26 19:17:05 mdejong Exp $ #----------------------------------------------------------------------- -# Sample configure.in for Tcl Extensions. The only places you should +# Sample configure.ac for Tcl Extensions. The only places you should # need to modify this file are marked by the string __CHANGE__ #----------------------------------------------------------------------- @@ -17,9 +15,11 @@ dnl to configure the system for the local environment. # This initializes the environment with PACKAGE_NAME and PACKAGE_VERSION # set as provided. These will also be added as -D defs in your Makefile # so you can encode the package version directly into the source files. +# This will also define a special symbol for Windows (BUILD_ +# so that we create the export library with the dll. #----------------------------------------------------------------------- -AC_INIT([sqlite], [3.32.0]) +AC_INIT([sqlite],[3.40.0]) #-------------------------------------------------------------------- # Call TEA_INIT as the first TEA_ macro to set up initial vars. @@ -27,7 +27,7 @@ AC_INIT([sqlite], [3.32.0]) # as well as PKG_LIB_FILE and PKG_STUB_LIB_FILE. #-------------------------------------------------------------------- -TEA_INIT([3.9]) +TEA_INIT() AC_CONFIG_AUX_DIR(tclconfig) @@ -55,8 +55,8 @@ TEA_PREFIX #----------------------------------------------------------------------- # Standard compiler checks. # This sets up CC by using the CC env var, or looks for gcc otherwise. -# This also calls AC_PROG_CC, AC_PROG_INSTALL and a few others to create -# the basic setup necessary to compile executables. +# This also calls AC_PROG_CC and a few others to create the basic setup +# necessary to compile executables. #----------------------------------------------------------------------- TEA_SETUP_COMPILER @@ -73,7 +73,7 @@ TEA_SETUP_COMPILER TEA_ADD_SOURCES([tclsqlite3.c]) TEA_ADD_HEADERS([]) -TEA_ADD_INCLUDES([-I\"`\${CYGPATH} \${srcdir}/generic`\"]) +TEA_ADD_INCLUDES([]) TEA_ADD_LIBS([]) TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1]) TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1]) @@ -99,6 +99,31 @@ if test x$with_system_sqlite != xno; then LIBS="$LIBS -lsqlite3"])]) fi +#-------------------------------------------------------------------- +# __CHANGE__ +# +# You can add more files to clean if your extension creates any extra +# files by extending CLEANFILES. +# Add pkgIndex.tcl if it is generated in the Makefile instead of ./configure +# and change Makefile.in to move it from CONFIG_CLEAN_FILES to BINARIES var. +# +# A few miscellaneous platform-specific items: +# TEA_ADD_* any platform specific compiler/build info here. +#-------------------------------------------------------------------- + +#CLEANFILES="$CLEANFILES pkgIndex.tcl" +if test "${TEA_PLATFORM}" = "windows" ; then + # Ensure no empty if clauses + : + #TEA_ADD_SOURCES([win/winFile.c]) + #TEA_ADD_INCLUDES([-I\"$(${CYGPATH} ${srcdir}/win)\"]) +else + # Ensure no empty else clauses + : + #TEA_ADD_SOURCES([unix/unixFile.c]) + #TEA_ADD_LIBS([-lsuperfly]) +fi + #-------------------------------------------------------------------- # __CHANGE__ # Choose which headers you need. Extension authors should try very @@ -151,28 +176,6 @@ TEA_CONFIG_CFLAGS TEA_ENABLE_SYMBOLS -#-------------------------------------------------------------------- -# Everyone should be linking against the Tcl stub library. If you -# can't for some reason, remove this definition. If you aren't using -# stubs, you also need to modify the SHLIB_LD_LIBS setting below to -# link against the non-stubbed Tcl library. Add Tk too if necessary. -#-------------------------------------------------------------------- - -AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs]) -#AC_DEFINE(USE_TK_STUBS, 1, [Use Tk stubs]) - - -#-------------------------------------------------------------------- -# Redefine fdatasync as fsync on systems that lack fdatasync -#-------------------------------------------------------------------- -# -#AC_CHECK_FUNC(fdatasync, , AC_DEFINE(fdatasync, fsync)) -# Check for library functions that SQLite can optionally use. -AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r]) - -AC_FUNC_STRERROR_R - - #-------------------------------------------------------------------- # This macro generates a line to use when building a library. It # depends on values set by the TEA_ENABLE_SHARED, TEA_ENABLE_SYMBOLS, @@ -193,9 +196,24 @@ TEA_PROG_TCLSH #TEA_PROG_WISH #-------------------------------------------------------------------- -# Finally, substitute all of the various values into the Makefile. -# You may alternatively have a special pkgIndex.tcl.in or other files -# which require substituting th AC variables in. Include these here. +# Setup a *Config.sh.in configuration file. #-------------------------------------------------------------------- -AC_OUTPUT([Makefile pkgIndex.tcl]) +#TEA_EXPORT_CONFIG([sample]) +#AC_SUBST(SAMPLE_VAR) + +#-------------------------------------------------------------------- +# Specify files to substitute AC variables in. You may alternatively +# have a special pkgIndex.tcl.in or other files which require +# substituting the AC variables in. Include these here. +#-------------------------------------------------------------------- + +AC_CONFIG_FILES([Makefile pkgIndex.tcl]) +#AC_CONFIG_FILES([sampleConfig.sh]) + +#-------------------------------------------------------------------- +# Finally, substitute all of the various values into the files +# specified with AC_CONFIG_FILES. +#-------------------------------------------------------------------- + +AC_OUTPUT diff --git a/autoconf/tea/pkgIndex.tcl.in b/autoconf/tea/pkgIndex.tcl.in index bc585f73b3..f95f7d3893 100644 --- a/autoconf/tea/pkgIndex.tcl.in +++ b/autoconf/tea/pkgIndex.tcl.in @@ -1,7 +1,10 @@ +# -*- tcl -*- +# Tcl package index file, version 1.1 # -# Tcl package index file -# -# Note sqlite*3* init specifically -# -package ifneeded sqlite3 @PACKAGE_VERSION@ \ - [list load [file join $dir @PKG_LIB_FILE@] Sqlite3] +if {[package vsatisfies [package provide Tcl] 9.0-]} { + package ifneeded sqlite3 @PACKAGE_VERSION@ \ + [list load [file join $dir @PKG_LIB_FILE9@] sqlite3] +} else { + package ifneeded sqlite3 @PACKAGE_VERSION@ \ + [list load [file join $dir @PKG_LIB_FILE8@] sqlite3] +} diff --git a/autoconf/tea/tclconfig/tcl.m4 b/autoconf/tea/tclconfig/tcl.m4 index 4b4bd1e888..c83d660e18 100644 --- a/autoconf/tea/tclconfig/tcl.m4 +++ b/autoconf/tea/tclconfig/tcl.m4 @@ -9,16 +9,13 @@ # See the file "license.terms" for information on usage and redistribution # of this file, and for a DISCLAIMER OF ALL WARRANTIES. -AC_PREREQ(2.57) - -dnl TEA extensions pass us the version of TEA they think they -dnl are compatible with (must be set in TEA_INIT below) -dnl TEA_VERSION="3.9" +AC_PREREQ([2.69]) # Possible values for key variables defined: # # TEA_WINDOWINGSYSTEM - win32 aqua x11 (mirrors 'tk windowingsystem') # TEA_PLATFORM - windows unix +# TEA_TK_EXTENSION - True if this is a Tk extension # #------------------------------------------------------------------------ @@ -53,9 +50,9 @@ AC_DEFUN([TEA_PATH_TCLCONFIG], [ # we reset no_tcl in case something fails here no_tcl=true AC_ARG_WITH(tcl, - AC_HELP_STRING([--with-tcl], + AS_HELP_STRING([--with-tcl], [directory containing tcl configuration (tclConfig.sh)]), - with_tclconfig="${withval}") + [with_tclconfig="${withval}"]) AC_MSG_CHECKING([for Tcl configuration]) AC_CACHE_VAL(ac_cv_c_tclconfig,[ @@ -107,7 +104,9 @@ AC_DEFUN([TEA_PATH_TCLCONFIG], [ for i in `ls -d ~/Library/Frameworks 2>/dev/null` \ `ls -d /Library/Frameworks 2>/dev/null` \ `ls -d /Network/Library/Frameworks 2>/dev/null` \ - `ls -d /System/Library/Frameworks 2>/dev/null` \ + `ls -d /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/Library/Frameworks/Tcl.framework 2>/dev/null` \ + `ls -d /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/Network/Library/Frameworks/Tcl.framework 2>/dev/null` \ + `ls -d /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks/Tcl.framework 2>/dev/null` \ ; do if test -f "$i/Tcl.framework/tclConfig.sh" ; then ac_cv_c_tclconfig="`(cd $i/Tcl.framework; pwd)`" @@ -136,10 +135,15 @@ AC_DEFUN([TEA_PATH_TCLCONFIG], [ `ls -d ${prefix}/lib 2>/dev/null` \ `ls -d /usr/local/lib 2>/dev/null` \ `ls -d /usr/contrib/lib 2>/dev/null` \ + `ls -d /usr/pkg/lib 2>/dev/null` \ `ls -d /usr/lib 2>/dev/null` \ `ls -d /usr/lib64 2>/dev/null` \ `ls -d /usr/lib/tcl8.6 2>/dev/null` \ `ls -d /usr/lib/tcl8.5 2>/dev/null` \ + `ls -d /usr/local/lib/tcl8.6 2>/dev/null` \ + `ls -d /usr/local/lib/tcl8.5 2>/dev/null` \ + `ls -d /usr/local/lib/tcl/tcl8.6 2>/dev/null` \ + `ls -d /usr/local/lib/tcl/tcl8.5 2>/dev/null` \ ; do if test -f "$i/tclConfig.sh" ; then ac_cv_c_tclconfig="`(cd $i; pwd)`" @@ -208,9 +212,9 @@ AC_DEFUN([TEA_PATH_TKCONFIG], [ # we reset no_tk in case something fails here no_tk=true AC_ARG_WITH(tk, - AC_HELP_STRING([--with-tk], + AS_HELP_STRING([--with-tk], [directory containing tk configuration (tkConfig.sh)]), - with_tkconfig="${withval}") + [with_tkconfig="${withval}"]) AC_MSG_CHECKING([for Tk configuration]) AC_CACHE_VAL(ac_cv_c_tkconfig,[ @@ -262,7 +266,6 @@ AC_DEFUN([TEA_PATH_TKCONFIG], [ for i in `ls -d ~/Library/Frameworks 2>/dev/null` \ `ls -d /Library/Frameworks 2>/dev/null` \ `ls -d /Network/Library/Frameworks 2>/dev/null` \ - `ls -d /System/Library/Frameworks 2>/dev/null` \ ; do if test -f "$i/Tk.framework/tkConfig.sh" ; then ac_cv_c_tkconfig="`(cd $i/Tk.framework; pwd)`" @@ -278,8 +281,15 @@ AC_DEFUN([TEA_PATH_TKCONFIG], [ `ls -d ${prefix}/lib 2>/dev/null` \ `ls -d /usr/local/lib 2>/dev/null` \ `ls -d /usr/contrib/lib 2>/dev/null` \ + `ls -d /usr/pkg/lib 2>/dev/null` \ + `ls -d /usr/lib/tk8.6 2>/dev/null` \ + `ls -d /usr/lib/tk8.5 2>/dev/null` \ `ls -d /usr/lib 2>/dev/null` \ `ls -d /usr/lib64 2>/dev/null` \ + `ls -d /usr/local/lib/tk8.6 2>/dev/null` \ + `ls -d /usr/local/lib/tk8.5 2>/dev/null` \ + `ls -d /usr/local/lib/tcl/tk8.6 2>/dev/null` \ + `ls -d /usr/local/lib/tcl/tk8.5 2>/dev/null` \ ; do if test -f "$i/tkConfig.sh" ; then ac_cv_c_tkconfig="`(cd $i; pwd)`" @@ -348,6 +358,8 @@ AC_DEFUN([TEA_PATH_TKCONFIG], [ # TCL_BIN_DIR # TCL_SRC_DIR # TCL_LIB_FILE +# TCL_ZIP_FILE +# TCL_ZIPFS_SUPPORT #------------------------------------------------------------------------ AC_DEFUN([TEA_LOAD_TCLCONFIG], [ @@ -360,10 +372,6 @@ AC_DEFUN([TEA_LOAD_TCLCONFIG], [ AC_MSG_RESULT([could not find ${TCL_BIN_DIR}/tclConfig.sh]) fi - # eval is required to do the TCL_DBGX substitution - eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\"" - eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\"" - # If the TCL_BIN_DIR is the build directory (not the install directory), # then set the common variable name to the value of the build variables. # For example, the variable TCL_LIB_SPEC will be set to the value @@ -397,12 +405,6 @@ AC_DEFUN([TEA_LOAD_TCLCONFIG], [ esac fi - # eval is required to do the TCL_DBGX substitution - eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\"" - eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\"" - eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\"" - eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\"" - AC_SUBST(TCL_VERSION) AC_SUBST(TCL_PATCH_LEVEL) AC_SUBST(TCL_BIN_DIR) @@ -418,13 +420,18 @@ AC_DEFUN([TEA_LOAD_TCLCONFIG], [ AC_MSG_CHECKING([platform]) hold_cc=$CC; CC="$TCL_CC" - AC_TRY_COMPILE(,[ + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ #ifdef _WIN32 #error win32 #endif - ], TEA_PLATFORM="unix", + ]])],[ + # first test we've already retrieved platform (cross-compile), fallback to unix otherwise: + TEA_PLATFORM="${TEA_PLATFORM-unix}" + CYGPATH=echo + ],[ TEA_PLATFORM="windows" - ) + AC_CHECK_PROG(CYGPATH, cygpath, cygpath -m, echo) + ]) CC=$hold_cc AC_MSG_RESULT($TEA_PLATFORM) @@ -473,10 +480,6 @@ AC_DEFUN([TEA_LOAD_TKCONFIG], [ AC_MSG_RESULT([could not find ${TK_BIN_DIR}/tkConfig.sh]) fi - # eval is required to do the TK_DBGX substitution - eval "TK_LIB_FILE=\"${TK_LIB_FILE}\"" - eval "TK_STUB_LIB_FILE=\"${TK_STUB_LIB_FILE}\"" - # If the TK_BIN_DIR is the build directory (not the install directory), # then set the common variable name to the value of the build variables. # For example, the variable TK_LIB_SPEC will be set to the value @@ -510,12 +513,6 @@ AC_DEFUN([TEA_LOAD_TKCONFIG], [ esac fi - # eval is required to do the TK_DBGX substitution - eval "TK_LIB_FLAG=\"${TK_LIB_FLAG}\"" - eval "TK_LIB_SPEC=\"${TK_LIB_SPEC}\"" - eval "TK_STUB_LIB_FLAG=\"${TK_STUB_LIB_FLAG}\"" - eval "TK_STUB_LIB_SPEC=\"${TK_STUB_LIB_SPEC}\"" - # TEA specific: Ensure windowingsystem is defined if test "${TEA_PLATFORM}" = "unix" ; then case ${TK_DEFS} in @@ -572,16 +569,24 @@ AC_DEFUN([TEA_PROG_TCLSH], [ if test -f "${TCL_BIN_DIR}/Makefile" ; then # tclConfig.sh is in Tcl build directory if test "${TEA_PLATFORM}" = "windows"; then - TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${TCL_DBGX}${EXEEXT}" + if test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" ; then + TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" + elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}s${EXEEXT}" ; then + TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}s${EXEEXT}" + elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}t${EXEEXT}" ; then + TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}t${EXEEXT}" + elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}st${EXEEXT}" ; then + TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}st${EXEEXT}" + fi else TCLSH_PROG="${TCL_BIN_DIR}/tclsh" fi else # tclConfig.sh is in install location if test "${TEA_PLATFORM}" = "windows"; then - TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${TCL_DBGX}${EXEEXT}" + TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" else - TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}.${TCL_MINOR_VERSION}${TCL_DBGX}" + TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}.${TCL_MINOR_VERSION}" fi list="`ls -d ${TCL_BIN_DIR}/../bin 2>/dev/null` \ `ls -d ${TCL_BIN_DIR}/.. 2>/dev/null` \ @@ -622,16 +627,24 @@ AC_DEFUN([TEA_PROG_WISH], [ if test -f "${TK_BIN_DIR}/Makefile" ; then # tkConfig.sh is in Tk build directory if test "${TEA_PLATFORM}" = "windows"; then - WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${TK_DBGX}${EXEEXT}" + if test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" ; then + WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" + elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}s${EXEEXT}" ; then + WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}$s{EXEEXT}" + elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}t${EXEEXT}" ; then + WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}t${EXEEXT}" + elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}st${EXEEXT}" ; then + WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}st${EXEEXT}" + fi else WISH_PROG="${TK_BIN_DIR}/wish" fi else # tkConfig.sh is in install location if test "${TEA_PLATFORM}" = "windows"; then - WISH_PROG="wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${TK_DBGX}${EXEEXT}" + WISH_PROG="wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" else - WISH_PROG="wish${TK_MAJOR_VERSION}.${TK_MINOR_VERSION}${TK_DBGX}" + WISH_PROG="wish${TK_MAJOR_VERSION}.${TK_MINOR_VERSION}" fi list="`ls -d ${TK_BIN_DIR}/../bin 2>/dev/null` \ `ls -d ${TK_BIN_DIR}/.. 2>/dev/null` \ @@ -660,6 +673,7 @@ AC_DEFUN([TEA_PROG_WISH], [ # # Adds the following arguments to configure: # --enable-shared=yes|no +# --enable-stubs=yes|no # # Defines the following vars: # STATIC_BUILD Used for building import/export libraries @@ -667,31 +681,63 @@ AC_DEFUN([TEA_PROG_WISH], [ # # Sets the following vars: # SHARED_BUILD Value of 1 or 0 +# STUBS_BUILD Value if 1 or 0 +# USE_TCL_STUBS Value true: if SHARED_BUILD or --enable-stubs +# USE_TCLOO_STUBS Value true: if SHARED_BUILD or --enable-stubs +# USE_TK_STUBS Value true: if SHARED_BUILD or --enable-stubs +# AND TEA_WINDOWING_SYSTEM != "" #------------------------------------------------------------------------ - AC_DEFUN([TEA_ENABLE_SHARED], [ AC_MSG_CHECKING([how to build libraries]) AC_ARG_ENABLE(shared, - AC_HELP_STRING([--enable-shared], + AS_HELP_STRING([--enable-shared], [build and link with shared libraries (default: on)]), - [tcl_ok=$enableval], [tcl_ok=yes]) + [shared_ok=$enableval], [shared_ok=yes]) if test "${enable_shared+set}" = set; then enableval="$enable_shared" - tcl_ok=$enableval + shared_ok=$enableval else - tcl_ok=yes + shared_ok=yes fi - if test "$tcl_ok" = "yes" ; then + AC_ARG_ENABLE(stubs, + AS_HELP_STRING([--enable-stubs], + [build and link with stub libraries. Always true for shared builds (default: on)]), + [stubs_ok=$enableval], [stubs_ok=yes]) + + if test "${enable_stubs+set}" = set; then + enableval="$enable_stubs" + stubs_ok=$enableval + else + stubs_ok=yes + fi + + # Stubs are always enabled for shared builds + if test "$shared_ok" = "yes" ; then AC_MSG_RESULT([shared]) SHARED_BUILD=1 + STUBS_BUILD=1 else AC_MSG_RESULT([static]) SHARED_BUILD=0 - AC_DEFINE(STATIC_BUILD, 1, [Is this a static build?]) + AC_DEFINE(STATIC_BUILD, 1, [This a static build]) + if test "$stubs_ok" = "yes" ; then + STUBS_BUILD=1 + else + STUBS_BUILD=0 + fi fi + if test "${STUBS_BUILD}" = "1" ; then + AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs]) + AC_DEFINE(USE_TCLOO_STUBS, 1, [Use TclOO stubs]) + if test "${TEA_WINDOWINGSYSTEM}" != ""; then + AC_DEFINE(USE_TK_STUBS, 1, [Use Tk stubs]) + fi + fi + AC_SUBST(SHARED_BUILD) + AC_SUBST(STUBS_BUILD) ]) #------------------------------------------------------------------------ @@ -728,8 +774,8 @@ AC_DEFUN([TEA_ENABLE_SHARED], [ AC_DEFUN([TEA_ENABLE_THREADS], [ AC_ARG_ENABLE(threads, - AC_HELP_STRING([--enable-threads], - [build with threads]), + AS_HELP_STRING([--enable-threads], + [build with threads (default: on)]), [tcl_ok=$enableval], [tcl_ok=yes]) if test "${enable_threads+set}" = set; then @@ -813,14 +859,6 @@ AC_DEFUN([TEA_ENABLE_THREADS], [ that IS thread-enabled. It is recommended to use --enable-threads.]) fi ;; - *) - if test "${TCL_THREADS}" = "1"; then - AC_MSG_WARN([ - --enable-threads requested, but building against a Tcl that is NOT - thread-enabled. This is an OK configuration that will also run in - a thread-enabled core.]) - fi - ;; esac AC_SUBST(TCL_THREADS) ]) @@ -850,8 +888,6 @@ AC_DEFUN([TEA_ENABLE_THREADS], [ # Sets to "$(CFLAGS_OPTIMIZE) -DNDEBUG" if false # LDFLAGS_DEFAULT Sets to $(LDFLAGS_DEBUG) if true # Sets to $(LDFLAGS_OPTIMIZE) if false -# DBGX Formerly used as debug library extension; -# always blank now. #------------------------------------------------------------------------ AC_DEFUN([TEA_ENABLE_SYMBOLS], [ @@ -859,14 +895,14 @@ AC_DEFUN([TEA_ENABLE_SYMBOLS], [ AC_REQUIRE([TEA_CONFIG_CFLAGS]) AC_MSG_CHECKING([for build with symbols]) AC_ARG_ENABLE(symbols, - AC_HELP_STRING([--enable-symbols], + AS_HELP_STRING([--enable-symbols], [build with debugging symbols (default: off)]), [tcl_ok=$enableval], [tcl_ok=no]) - DBGX="" if test "$tcl_ok" = "no"; then CFLAGS_DEFAULT="${CFLAGS_OPTIMIZE} -DNDEBUG" LDFLAGS_DEFAULT="${LDFLAGS_OPTIMIZE}" AC_MSG_RESULT([no]) + AC_DEFINE(TCL_CFG_OPTIMIZED, 1, [Is this an optimized build?]) else CFLAGS_DEFAULT="${CFLAGS_DEBUG}" LDFLAGS_DEFAULT="${LDFLAGS_DEBUG}" @@ -874,13 +910,8 @@ AC_DEFUN([TEA_ENABLE_SYMBOLS], [ AC_MSG_RESULT([yes (standard debugging)]) fi fi - # TEA specific: - if test "${TEA_PLATFORM}" != "windows" ; then - LDFLAGS_DEFAULT="${LDFLAGS}" - fi AC_SUBST(CFLAGS_DEFAULT) AC_SUBST(LDFLAGS_DEFAULT) - AC_SUBST(TCL_DBGX) if test "$tcl_ok" = "mem" -o "$tcl_ok" = "all"; then AC_DEFINE(TCL_MEM_DEBUG, 1, [Is memory debugging enabled?]) @@ -915,7 +946,7 @@ AC_DEFUN([TEA_ENABLE_SYMBOLS], [ AC_DEFUN([TEA_ENABLE_LANGINFO], [ AC_ARG_ENABLE(langinfo, - AC_HELP_STRING([--enable-langinfo], + AS_HELP_STRING([--enable-langinfo], [use nl_langinfo if possible to determine encoding at startup, otherwise use old heuristic (default: on)]), [langinfo_ok=$enableval], [langinfo_ok=yes]) @@ -926,7 +957,7 @@ AC_DEFUN([TEA_ENABLE_LANGINFO], [ AC_MSG_CHECKING([whether to use nl_langinfo]) if test "$langinfo_ok" = "yes"; then AC_CACHE_VAL(tcl_cv_langinfo_h, [ - AC_TRY_COMPILE([#include ], [nl_langinfo(CODESET);], + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[nl_langinfo(CODESET);]])], [tcl_cv_langinfo_h=yes],[tcl_cv_langinfo_h=no])]) AC_MSG_RESULT([$tcl_cv_langinfo_h]) if test $tcl_cv_langinfo_h = yes; then @@ -951,6 +982,7 @@ AC_DEFUN([TEA_ENABLE_LANGINFO], [ # Defines the following var: # # system - System/platform/version identification code. +# #-------------------------------------------------------------------- AC_DEFUN([TEA_CONFIG_SYSTEM], [ @@ -967,6 +999,9 @@ AC_DEFUN([TEA_CONFIG_SYSTEM], [ if test "`uname -s`" = "AIX" ; then tcl_cv_sys_version=AIX-`uname -v`.`uname -r` fi + if test "`uname -s`" = "NetBSD" -a -f /etc/debian_version ; then + tcl_cv_sys_version=NetBSD-Debian + fi fi fi ]) @@ -1043,7 +1078,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ AC_MSG_CHECKING([if 64bit support is requested]) AC_ARG_ENABLE(64bit, - AC_HELP_STRING([--enable-64bit], + AS_HELP_STRING([--enable-64bit], [enable 64bit support (default: off)]), [do64bit=$enableval], [do64bit=no]) AC_MSG_RESULT([$do64bit]) @@ -1052,7 +1087,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ AC_MSG_CHECKING([if 64bit Sparc VIS support is requested]) AC_ARG_ENABLE(64bit-vis, - AC_HELP_STRING([--enable-64bit-vis], + AS_HELP_STRING([--enable-64bit-vis], [enable 64bit Sparc VIS support (default: off)]), [do64bitVIS=$enableval], [do64bitVIS=no]) AC_MSG_RESULT([$do64bitVIS]) @@ -1065,10 +1100,10 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ AC_CACHE_CHECK([if compiler supports visibility "hidden"], tcl_cv_cc_visibility_hidden, [ hold_cflags=$CFLAGS; CFLAGS="$CFLAGS -Werror" - AC_TRY_LINK([ + AC_LINK_IFELSE([AC_LANG_PROGRAM([[ extern __attribute__((__visibility__("hidden"))) void f(void); - void f(void) {}], [f();], tcl_cv_cc_visibility_hidden=yes, - tcl_cv_cc_visibility_hidden=no) + void f(void) {}]], [[f();]])],[tcl_cv_cc_visibility_hidden=yes], + [tcl_cv_cc_visibility_hidden=no]) CFLAGS=$hold_cflags]) AS_IF([test $tcl_cv_cc_visibility_hidden = yes], [ AC_DEFINE(MODULE_SCOPE, @@ -1081,22 +1116,11 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ AC_MSG_CHECKING([if rpath support is requested]) AC_ARG_ENABLE(rpath, - AC_HELP_STRING([--disable-rpath], + AS_HELP_STRING([--disable-rpath], [disable rpath support (default: on)]), [doRpath=$enableval], [doRpath=yes]) AC_MSG_RESULT([$doRpath]) - # TEA specific: Cross-compiling options for Windows/CE builds? - - AS_IF([test "${TEA_PLATFORM}" = windows], [ - AC_MSG_CHECKING([if Windows/CE build is requested]) - AC_ARG_ENABLE(wince, - AC_HELP_STRING([--enable-wince], - [enable Win/CE support (where applicable)]), - [doWince=$enableval], [doWince=no]) - AC_MSG_RESULT([$doWince]) - ]) - # Set the variable "system" to hold the name and version number # for the system. @@ -1133,99 +1157,23 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ AC_CHECK_TOOL(AR, ar) STLIB_LD='${AR} cr' LD_LIBRARY_PATH_VAR="LD_LIBRARY_PATH" - AS_IF([test "x$SHLIB_VERSION" = x],[SHLIB_VERSION="1.0"]) + AS_IF([test "x$SHLIB_VERSION" = x],[SHLIB_VERSION=""],[SHLIB_VERSION=".$SHLIB_VERSION"]) case $system in # TEA specific: windows) - # This is a 2-stage check to make sure we have the 64-bit SDK - # We have to know where the SDK is installed. - # This magic is based on MS Platform SDK for Win2003 SP1 - hobbs - # MACHINE is IX86 for LINK, but this is used by the manifest, - # which requires x86|amd64|ia64. MACHINE="X86" if test "$do64bit" != "no" ; then - if test "x${MSSDK}x" = "xx" ; then - MSSDK="C:/Progra~1/Microsoft Platform SDK" - fi - MSSDK=`echo "$MSSDK" | sed -e 's!\\\!/!g'` - PATH64="" case "$do64bit" in amd64|x64|yes) MACHINE="AMD64" ; # default to AMD64 64-bit build - PATH64="${MSSDK}/Bin/Win64/x86/AMD64" + ;; + arm64|aarch64) + MACHINE="ARM64" ;; ia64) MACHINE="IA64" - PATH64="${MSSDK}/Bin/Win64" ;; esac - if test "$GCC" != "yes" -a ! -d "${PATH64}" ; then - AC_MSG_WARN([Could not find 64-bit $MACHINE SDK to enable 64bit mode]) - AC_MSG_WARN([Ensure latest Platform SDK is installed]) - do64bit="no" - else - AC_MSG_RESULT([ Using 64-bit $MACHINE mode]) - do64bit_ok="yes" - fi - fi - - if test "$doWince" != "no" ; then - if test "$do64bit" != "no" ; then - AC_MSG_ERROR([Windows/CE and 64-bit builds incompatible]) - fi - if test "$GCC" = "yes" ; then - AC_MSG_ERROR([Windows/CE and GCC builds incompatible]) - fi - TEA_PATH_CELIB - # Set defaults for common evc4/PPC2003 setup - # Currently Tcl requires 300+, possibly 420+ for sockets - CEVERSION=420; # could be 211 300 301 400 420 ... - TARGETCPU=ARMV4; # could be ARMV4 ARM MIPS SH3 X86 ... - ARCH=ARM; # could be ARM MIPS X86EM ... - PLATFORM="Pocket PC 2003"; # or "Pocket PC 2002" - if test "$doWince" != "yes"; then - # If !yes then the user specified something - # Reset ARCH to allow user to skip specifying it - ARCH= - eval `echo $doWince | awk -F, '{ \ - if (length([$]1)) { printf "CEVERSION=\"%s\"\n", [$]1; \ - if ([$]1 < 400) { printf "PLATFORM=\"Pocket PC 2002\"\n" } }; \ - if (length([$]2)) { printf "TARGETCPU=\"%s\"\n", toupper([$]2) }; \ - if (length([$]3)) { printf "ARCH=\"%s\"\n", toupper([$]3) }; \ - if (length([$]4)) { printf "PLATFORM=\"%s\"\n", [$]4 }; \ - }'` - if test "x${ARCH}" = "x" ; then - ARCH=$TARGETCPU; - fi - fi - OSVERSION=WCE$CEVERSION; - if test "x${WCEROOT}" = "x" ; then - WCEROOT="C:/Program Files/Microsoft eMbedded C++ 4.0" - if test ! -d "${WCEROOT}" ; then - WCEROOT="C:/Program Files/Microsoft eMbedded Tools" - fi - fi - if test "x${SDKROOT}" = "x" ; then - SDKROOT="C:/Program Files/Windows CE Tools" - if test ! -d "${SDKROOT}" ; then - SDKROOT="C:/Windows CE Tools" - fi - fi - WCEROOT=`echo "$WCEROOT" | sed -e 's!\\\!/!g'` - SDKROOT=`echo "$SDKROOT" | sed -e 's!\\\!/!g'` - if test ! -d "${SDKROOT}/${OSVERSION}/${PLATFORM}/Lib/${TARGETCPU}" \ - -o ! -d "${WCEROOT}/EVC/${OSVERSION}/bin"; then - AC_MSG_ERROR([could not find PocketPC SDK or target compiler to enable WinCE mode [$CEVERSION,$TARGETCPU,$ARCH,$PLATFORM]]) - doWince="no" - else - # We could PATH_NOSPACE these, but that's not important, - # as long as we quote them when used. - CEINCLUDE="${SDKROOT}/${OSVERSION}/${PLATFORM}/include" - if test -d "${CEINCLUDE}/${TARGETCPU}" ; then - CEINCLUDE="${CEINCLUDE}/${TARGETCPU}" - fi - CELIBPATH="${SDKROOT}/${OSVERSION}/${PLATFORM}/Lib/${TARGETCPU}" - fi fi if test "$GCC" != "yes" ; then @@ -1234,48 +1182,28 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ else runtime=-MD fi + case "x`echo \${VisualStudioVersion}`" in + x1[[4-9]]*) + lflags="${lflags} -nodefaultlib:libucrt.lib" + TEA_ADD_LIBS([ucrt.lib]) + ;; + *) + ;; + esac if test "$do64bit" != "no" ; then - # All this magic is necessary for the Win64 SDK RC1 - hobbs - CC="\"${PATH64}/cl.exe\"" - CFLAGS="${CFLAGS} -I\"${MSSDK}/Include\" -I\"${MSSDK}/Include/crt\" -I\"${MSSDK}/Include/crt/sys\"" - RC="\"${MSSDK}/bin/rc.exe\"" - lflags="-nologo -MACHINE:${MACHINE} -LIBPATH:\"${MSSDK}/Lib/${MACHINE}\"" - LINKBIN="\"${PATH64}/link.exe\"" + CC="cl.exe" + RC="rc.exe" + lflags="${lflags} -nologo -MACHINE:${MACHINE} " + LINKBIN="link.exe" CFLAGS_DEBUG="-nologo -Zi -Od -W3 ${runtime}d" CFLAGS_OPTIMIZE="-nologo -O2 -W2 ${runtime}" # Avoid 'unresolved external symbol __security_cookie' # errors, c.f. http://support.microsoft.com/?id=894573 TEA_ADD_LIBS([bufferoverflowU.lib]) - elif test "$doWince" != "no" ; then - CEBINROOT="${WCEROOT}/EVC/${OSVERSION}/bin" - if test "${TARGETCPU}" = "X86"; then - CC="\"${CEBINROOT}/cl.exe\"" - else - CC="\"${CEBINROOT}/cl${ARCH}.exe\"" - fi - CFLAGS="$CFLAGS -I\"${CELIB_DIR}/inc\" -I\"${CEINCLUDE}\"" - RC="\"${WCEROOT}/Common/EVC/bin/rc.exe\"" - arch=`echo ${ARCH} | awk '{print tolower([$]0)}'` - defs="${ARCH} _${ARCH}_ ${arch} PALM_SIZE _MT _WINDOWS" - if test "${SHARED_BUILD}" = "1" ; then - # Static CE builds require static celib as well - defs="${defs} _DLL" - fi - for i in $defs ; do - AC_DEFINE_UNQUOTED($i, 1, [WinCE def ]$i) - done - AC_DEFINE_UNQUOTED(_WIN32_WCE, $CEVERSION, [_WIN32_WCE version]) - AC_DEFINE_UNQUOTED(UNDER_CE, $CEVERSION, [UNDER_CE version]) - CFLAGS_DEBUG="-nologo -Zi -Od" - CFLAGS_OPTIMIZE="-nologo -Ox" - lversion=`echo ${CEVERSION} | sed -e 's/\(.\)\(..\)/\1\.\2/'` - lflags="-MACHINE:${ARCH} -LIBPATH:\"${CELIBPATH}\" -subsystem:windowsce,${lversion} -nologo" - LINKBIN="\"${CEBINROOT}/link.exe\"" - AC_SUBST(CELIB_DIR) else RC="rc" - lflags="-nologo" + lflags="${lflags} -nologo" LINKBIN="link" CFLAGS_DEBUG="-nologo -Z7 -Od -W3 -WX ${runtime}d" CFLAGS_OPTIMIZE="-nologo -O2 -W2 ${runtime}" @@ -1294,25 +1222,32 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ AC_CACHE_CHECK(for cross-compile version of gcc, ac_cv_cross, - AC_TRY_COMPILE([ + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #ifdef _WIN32 #error cross-compiler #endif - ], [], - ac_cv_cross=yes, - ac_cv_cross=no) + ]], [[]])], + [ac_cv_cross=yes], + [ac_cv_cross=no]) ) if test "$ac_cv_cross" = "yes"; then case "$do64bit" in amd64|x64|yes) - CC="x86_64-w64-mingw32-gcc" + CC="x86_64-w64-mingw32-${CC}" LD="x86_64-w64-mingw32-ld" AR="x86_64-w64-mingw32-ar" RANLIB="x86_64-w64-mingw32-ranlib" RC="x86_64-w64-mingw32-windres" ;; + arm64|aarch64) + CC="aarch64-w64-mingw32-clang" + LD="aarch64-w64-mingw32-ld" + AR="aarch64-w64-mingw32-ar" + RANLIB="aarch64-w64-mingw32-ranlib" + RC="aarch64-w64-mingw32-windres" + ;; *) - CC="i686-w64-mingw32-gcc" + CC="i686-w64-mingw32-${CC}" LD="i686-w64-mingw32-ld" AR="i686-w64-mingw32-ar" RANLIB="i686-w64-mingw32-ranlib" @@ -1334,13 +1269,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ # This essentially turns it all on. LDFLAGS_DEBUG="-debug -debugtype:cv" LDFLAGS_OPTIMIZE="-release" - if test "$doWince" != "no" ; then - LDFLAGS_CONSOLE="-link ${lflags}" - LDFLAGS_WINDOW=${LDFLAGS_CONSOLE} - else - LDFLAGS_CONSOLE="-link -subsystem:console ${lflags}" - LDFLAGS_WINDOW="-link -subsystem:windows ${lflags}" - fi + LDFLAGS_CONSOLE="-link -subsystem:console ${lflags}" + LDFLAGS_WINDOW="-link -subsystem:windows ${lflags}" fi SHLIB_SUFFIX=".dll" @@ -1349,7 +1279,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ TCL_LIB_VERSIONS_OK=nodots ;; AIX-*) - AS_IF([test "${TCL_THREADS}" = "1" -a "$GCC" != "yes"], [ + AS_IF([test "$GCC" != "yes"], [ # AIX requires the _r compiler when gcc isn't being used case "${CC}" in *_r|*_r\ *) @@ -1386,11 +1316,11 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ # AIX-5 uses ELF style dynamic libraries on IA-64, but not PPC SHLIB_LD="/usr/ccs/bin/ld -G -z text" AS_IF([test "$GCC" = yes], [ - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' + CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' ], [ - CC_SEARCH_FLAGS='-R${LIB_RUNTIME_DIR}' + CC_SEARCH_FLAGS='"-R${LIB_RUNTIME_DIR}"' ]) - LD_SEARCH_FLAGS='-R ${LIB_RUNTIME_DIR}' + LD_SEARCH_FLAGS='-R "${LIB_RUNTIME_DIR}"' ], [ AS_IF([test "$GCC" = yes], [ SHLIB_LD='${CC} -shared -Wl,-bexpall' @@ -1399,7 +1329,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ LDFLAGS="$LDFLAGS -brtl" ]) SHLIB_LD="${SHLIB_LD} ${SHLIB_LD_FLAGS}" - CC_SEARCH_FLAGS='-L${LIB_RUNTIME_DIR}' + CC_SEARCH_FLAGS='"-L${LIB_RUNTIME_DIR}"' LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} ]) ;; @@ -1415,6 +1345,13 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ #----------------------------------------------------------- AC_CHECK_LIB(bind, inet_ntoa, [LIBS="$LIBS -lbind -lsocket"]) ;; + BSD/OS-2.1*|BSD/OS-3*) + SHLIB_CFLAGS="" + SHLIB_LD="shlicc -r" + SHLIB_SUFFIX=".so" + CC_SEARCH_FLAGS="" + LD_SEARCH_FLAGS="" + ;; BSD/OS-4.*) SHLIB_CFLAGS="-export-dynamic -fPIC" SHLIB_LD='${CC} -shared' @@ -1427,16 +1364,25 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ SHLIB_CFLAGS="" SHLIB_LD='${CC} -shared' SHLIB_SUFFIX=".dll" + SHLIB_LD_LIBS="${SHLIB_LD_LIBS} -Wl,--out-implib,\$[@].a" EXEEXT=".exe" do64bit_ok=yes CC_SEARCH_FLAGS="" LD_SEARCH_FLAGS="" ;; + dgux*) + SHLIB_CFLAGS="-K PIC" + SHLIB_LD='${CC} -G' + SHLIB_LD_LIBS="" + SHLIB_SUFFIX=".so" + CC_SEARCH_FLAGS="" + LD_SEARCH_FLAGS="" + ;; Haiku*) LDFLAGS="$LDFLAGS -Wl,--export-dynamic" SHLIB_CFLAGS="-fPIC" SHLIB_SUFFIX=".so" - SHLIB_LD='${CC} -shared ${CFLAGS} ${LDFLAGS}' + SHLIB_LD='${CC} ${CFLAGS} ${LDFLAGS} -shared' AC_CHECK_LIB(network, inet_ntoa, [LIBS="$LIBS -lnetwork"]) ;; HP-UX-*.11.*) @@ -1448,18 +1394,16 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ AS_IF([test "`uname -m`" = ia64], [ SHLIB_SUFFIX=".so" - # Use newer C++ library for C++ extensions - #if test "$GCC" != "yes" ; then - # CPPFLAGS="-AA" - #fi ], [ SHLIB_SUFFIX=".sl" ]) AC_CHECK_LIB(dld, shl_load, tcl_ok=yes, tcl_ok=no) AS_IF([test "$tcl_ok" = yes], [ + SHLIB_CFLAGS="+z" + SHLIB_LD="ld -b" LDFLAGS="$LDFLAGS -Wl,-E" - CC_SEARCH_FLAGS='-Wl,+s,+b,${LIB_RUNTIME_DIR}:.' - LD_SEARCH_FLAGS='+s +b ${LIB_RUNTIME_DIR}:.' + CC_SEARCH_FLAGS='"-Wl,+s,+b,${LIB_RUNTIME_DIR}:."' + LD_SEARCH_FLAGS='+s +b "${LIB_RUNTIME_DIR}:."' LD_LIBRARY_PATH_VAR="SHLIB_PATH" ]) AS_IF([test "$GCC" = yes], [ @@ -1467,10 +1411,6 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} ], [ CFLAGS="$CFLAGS -z" - # Users may want PA-RISC 1.1/2.0 portable code - needs HP cc - #CFLAGS="$CFLAGS +DAportable" - SHLIB_CFLAGS="+z" - SHLIB_LD="ld -b" ]) # Check to enable 64-bit flags for compiler/linker @@ -1482,7 +1422,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ do64bit_ok=yes SHLIB_LD='${CC} -shared' AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) + CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} ;; *) @@ -1495,13 +1435,34 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ LDFLAGS_ARCH="+DD64" ]) ]) ;; + HP-UX-*.08.*|HP-UX-*.09.*|HP-UX-*.10.*) + SHLIB_SUFFIX=".sl" + AC_CHECK_LIB(dld, shl_load, tcl_ok=yes, tcl_ok=no) + AS_IF([test "$tcl_ok" = yes], [ + SHLIB_CFLAGS="+z" + SHLIB_LD="ld -b" + SHLIB_LD_LIBS="" + LDFLAGS="$LDFLAGS -Wl,-E" + CC_SEARCH_FLAGS='"-Wl,+s,+b,${LIB_RUNTIME_DIR}:."' + LD_SEARCH_FLAGS='+s +b "${LIB_RUNTIME_DIR}:."' + LD_LIBRARY_PATH_VAR="SHLIB_PATH" + ]) ;; + IRIX-5.*) + SHLIB_CFLAGS="" + SHLIB_LD="ld -shared -rdata_shared" + SHLIB_SUFFIX=".so" + AC_LIBOBJ(mkstemp) + AS_IF([test $doRpath = yes], [ + CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' + LD_SEARCH_FLAGS='-rpath "${LIB_RUNTIME_DIR}"']) + ;; IRIX-6.*) SHLIB_CFLAGS="" SHLIB_LD="ld -n32 -shared -rdata_shared" SHLIB_SUFFIX=".so" AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) + CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' + LD_SEARCH_FLAGS='-rpath "${LIB_RUNTIME_DIR}"']) AS_IF([test "$GCC" = yes], [ CFLAGS="$CFLAGS -mabi=n32" LDFLAGS="$LDFLAGS -mabi=n32" @@ -1523,8 +1484,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ SHLIB_LD="ld -n32 -shared -rdata_shared" SHLIB_SUFFIX=".so" AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) + CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' + LD_SEARCH_FLAGS='-rpath "${LIB_RUNTIME_DIR}"']) # Check to enable 64-bit flags for compiler/linker @@ -1539,7 +1500,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ ]) ]) ;; - Linux*|GNU*|NetBSD-Debian) + Linux*|GNU*|NetBSD-Debian|DragonFly-*|FreeBSD-*) SHLIB_CFLAGS="-fPIC" SHLIB_SUFFIX=".so" @@ -1547,17 +1508,29 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ CFLAGS_OPTIMIZE="-O2 -fomit-frame-pointer" # TEA specific: use LDFLAGS_DEFAULT instead of LDFLAGS - SHLIB_LD='${CC} -shared ${CFLAGS} ${LDFLAGS_DEFAULT}' + SHLIB_LD='${CC} ${CFLAGS} ${LDFLAGS_DEFAULT} -shared' LDFLAGS="$LDFLAGS -Wl,--export-dynamic" + + case $system in + DragonFly-*|FreeBSD-*) + AS_IF([test "${TCL_THREADS}" = "1"], [ + # The -pthread needs to go in the LDFLAGS, not LIBS + LIBS=`echo $LIBS | sed s/-pthread//` + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + LDFLAGS="$LDFLAGS $PTHREAD_LIBS"]) + ;; + esac + AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) + CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} AS_IF([test "`uname -m`" = "alpha"], [CFLAGS="$CFLAGS -mieee"]) AS_IF([test $do64bit = yes], [ AC_CACHE_CHECK([if compiler accepts -m64 flag], tcl_cv_cc_m64, [ hold_cflags=$CFLAGS CFLAGS="$CFLAGS -m64" - AC_TRY_LINK(,, tcl_cv_cc_m64=yes, tcl_cv_cc_m64=no) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])], + [tcl_cv_cc_m64=yes],[tcl_cv_cc_m64=no]) CFLAGS=$hold_cflags]) AS_IF([test $tcl_cv_cc_m64 = yes], [ CFLAGS="$CFLAGS -m64" @@ -1580,42 +1553,31 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ SHLIB_LD='${CC} -shared' LD_FLAGS="-Wl,--export-dynamic" AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) + CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' + LD_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) ;; OpenBSD-*) arch=`arch -s` case "$arch" in - vax) - SHLIB_SUFFIX="" - SHARED_LIB_SUFFIX="" - LDFLAGS="" - ;; - *) + alpha|sparc64) SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} -shared ${SHLIB_CFLAGS}' - SHLIB_SUFFIX=".so" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so.${SHLIB_VERSION}' - LDFLAGS="-Wl,-export-dynamic" - ;; - esac - case "$arch" in - vax) - CFLAGS_OPTIMIZE="-O1" ;; *) - CFLAGS_OPTIMIZE="-O2" + SHLIB_CFLAGS="-fpic" ;; esac - AS_IF([test "${TCL_THREADS}" = "1"], [ - # On OpenBSD: Compile with -pthread - # Don't link with -lpthread - LIBS=`echo $LIBS | sed s/-lpthread//` - CFLAGS="$CFLAGS -pthread" - ]) + SHLIB_LD='${CC} ${SHLIB_CFLAGS} -shared' + SHLIB_SUFFIX=".so" + AS_IF([test $doRpath = yes], [ + CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) + LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} + SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so${SHLIB_VERSION}' + LDFLAGS="$LDFLAGS -Wl,-export-dynamic" + CFLAGS_OPTIMIZE="-O2" + # On OpenBSD: Compile with -pthread + # Don't link with -lpthread + LIBS=`echo $LIBS | sed s/-lpthread//` + CFLAGS="$CFLAGS -pthread" # OpenBSD doesn't do version numbers with dots. UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' TCL_LIB_VERSIONS_OK=nodots @@ -1623,44 +1585,16 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ NetBSD-*) # NetBSD has ELF and can use 'cc -shared' to build shared libs SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} -shared ${SHLIB_CFLAGS}' + SHLIB_LD='${CC} ${SHLIB_CFLAGS} -shared' SHLIB_SUFFIX=".so" LDFLAGS="$LDFLAGS -export-dynamic" AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) + CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "${TCL_THREADS}" = "1"], [ - # The -pthread needs to go in the CFLAGS, not LIBS - LIBS=`echo $LIBS | sed s/-pthread//` - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - ]) - ;; - FreeBSD-*) - # This configuration from FreeBSD Ports. - SHLIB_CFLAGS="-fPIC" - SHLIB_LD="${CC} -shared" - TCL_SHLIB_LD_EXTRAS="-Wl,-soname=\$[@]" - TK_SHLIB_LD_EXTRAS="-Wl,-soname,\$[@]" - SHLIB_SUFFIX=".so" - LDFLAGS="" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}']) - AS_IF([test "${TCL_THREADS}" = "1"], [ - # The -pthread needs to go in the LDFLAGS, not LIBS - LIBS=`echo $LIBS | sed s/-pthread//` - CFLAGS="$CFLAGS $PTHREAD_CFLAGS" - LDFLAGS="$LDFLAGS $PTHREAD_LIBS"]) - case $system in - FreeBSD-3.*) - # Version numbers are dot-stripped by system policy. - TCL_TRIM_DOTS=`echo ${VERSION} | tr -d .` - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so' - TCL_LIB_VERSIONS_OK=nodots - ;; - esac + # The -pthread needs to go in the CFLAGS, not LIBS + LIBS=`echo $LIBS | sed s/-pthread//` + CFLAGS="$CFLAGS -pthread" + LDFLAGS="$LDFLAGS -pthread" ;; Darwin-*) CFLAGS_OPTIMIZE="-Os" @@ -1681,8 +1615,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ tcl_cv_cc_arch_ppc64, [ hold_cflags=$CFLAGS CFLAGS="$CFLAGS -arch ppc64 -mpowerpc64 -mcpu=G5" - AC_TRY_LINK(,, tcl_cv_cc_arch_ppc64=yes, - tcl_cv_cc_arch_ppc64=no) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])], + [tcl_cv_cc_arch_ppc64=yes],[tcl_cv_cc_arch_ppc64=no]) CFLAGS=$hold_cflags]) AS_IF([test $tcl_cv_cc_arch_ppc64 = yes], [ CFLAGS="$CFLAGS -arch ppc64 -mpowerpc64 -mcpu=G5" @@ -1693,8 +1627,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ tcl_cv_cc_arch_x86_64, [ hold_cflags=$CFLAGS CFLAGS="$CFLAGS -arch x86_64" - AC_TRY_LINK(,, tcl_cv_cc_arch_x86_64=yes, - tcl_cv_cc_arch_x86_64=no) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])], + [tcl_cv_cc_arch_x86_64=yes],[tcl_cv_cc_arch_x86_64=no]) CFLAGS=$hold_cflags]) AS_IF([test $tcl_cv_cc_arch_x86_64 = yes], [ CFLAGS="$CFLAGS -arch x86_64" @@ -1714,7 +1648,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ AC_CACHE_CHECK([if ld accepts -single_module flag], tcl_cv_ld_single_module, [ hold_ldflags=$LDFLAGS LDFLAGS="$LDFLAGS -dynamiclib -Wl,-single_module" - AC_TRY_LINK(, [int i;], tcl_cv_ld_single_module=yes, tcl_cv_ld_single_module=no) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[int i;]])], + [tcl_cv_ld_single_module=yes],[tcl_cv_ld_single_module=no]) LDFLAGS=$hold_ldflags]) AS_IF([test $tcl_cv_ld_single_module = yes], [ SHLIB_LD="${SHLIB_LD} -Wl,-single_module" @@ -1723,17 +1658,13 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ vers=`echo ${PACKAGE_VERSION} | sed -e 's/^\([[0-9]]\{1,5\}\)\(\(\.[[0-9]]\{1,3\}\)\{0,2\}\).*$/\1\2/p' -e d` SHLIB_LD="${SHLIB_LD} -current_version ${vers:-0} -compatibility_version ${vers:-0}" SHLIB_SUFFIX=".dylib" - # Don't use -prebind when building for Mac OS X 10.4 or later only: - AS_IF([test "`echo "${MACOSX_DEPLOYMENT_TARGET}" | awk -F '10\\.' '{print int([$]2)}'`" -lt 4 -a \ - "`echo "${CPPFLAGS}" | awk -F '-mmacosx-version-min=10\\.' '{print int([$]2)}'`" -lt 4], [ - LDFLAGS="$LDFLAGS -prebind"]) LDFLAGS="$LDFLAGS -headerpad_max_install_names" AC_CACHE_CHECK([if ld accepts -search_paths_first flag], tcl_cv_ld_search_paths_first, [ hold_ldflags=$LDFLAGS LDFLAGS="$LDFLAGS -Wl,-search_paths_first" - AC_TRY_LINK(, [int i;], tcl_cv_ld_search_paths_first=yes, - tcl_cv_ld_search_paths_first=no) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[int i;]])], + [tcl_cv_ld_search_paths_first=yes],[tcl_cv_ld_search_paths_first=no]) LDFLAGS=$hold_ldflags]) AS_IF([test $tcl_cv_ld_search_paths_first = yes], [ LDFLAGS="$LDFLAGS -Wl,-search_paths_first" @@ -1756,8 +1687,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ done CPPFLAGS="$CPPFLAGS -I/usr/X11R6/include" LDFLAGS="$LDFLAGS -L/usr/X11R6/lib -lX11" - AC_TRY_LINK([#include ], [XrmInitialize();], - tcl_cv_lib_x11_64=yes, tcl_cv_lib_x11_64=no) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], [[XrmInitialize();]])], + [tcl_cv_lib_x11_64=yes],[tcl_cv_lib_x11_64=no]) for v in CFLAGS CPPFLAGS LDFLAGS; do eval $v'="$hold_'$v'"' done]) @@ -1769,8 +1700,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ done CPPFLAGS="$CPPFLAGS -DUSE_TCL_STUBS=1 -DUSE_TK_STUBS=1 ${TCL_INCLUDES} ${TK_INCLUDES}" LDFLAGS="$LDFLAGS ${TCL_STUB_LIB_SPEC} ${TK_STUB_LIB_SPEC}" - AC_TRY_LINK([#include ], [Tk_InitStubs(NULL, "", 0);], - tcl_cv_lib_tk_64=yes, tcl_cv_lib_tk_64=no) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], [[Tk_InitStubs(NULL, "", 0);]])], + [tcl_cv_lib_tk_64=yes],[tcl_cv_lib_tk_64=no]) for v in CFLAGS CPPFLAGS LDFLAGS; do eval $v'="$hold_'$v'"' done]) @@ -1799,21 +1730,19 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ ]) SHLIB_SUFFIX=".so" AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='-Wl,-rpath,${LIB_RUNTIME_DIR}' + CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) AS_IF([test "$GCC" = yes], [CFLAGS="$CFLAGS -mieee"], [ CFLAGS="$CFLAGS -DHAVE_TZSET -std1 -ieee"]) # see pthread_intro(3) for pthread support on osf1, k.furukawa - AS_IF([test "${TCL_THREADS}" = 1], [ - CFLAGS="$CFLAGS -DHAVE_PTHREAD_ATTR_SETSTACKSIZE" - CFLAGS="$CFLAGS -DTCL_THREAD_STACK_MIN=PTHREAD_STACK_MIN*64" - LIBS=`echo $LIBS | sed s/-lpthreads//` - AS_IF([test "$GCC" = yes], [ - LIBS="$LIBS -lpthread -lmach -lexc" - ], [ - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - ]) + CFLAGS="$CFLAGS -DHAVE_PTHREAD_ATTR_SETSTACKSIZE" + CFLAGS="$CFLAGS -DTCL_THREAD_STACK_MIN=PTHREAD_STACK_MIN*64" + LIBS=`echo $LIBS | sed s/-lpthreads//` + AS_IF([test "$GCC" = yes], [ + LIBS="$LIBS -lpthread -lmach -lexc" + ], [ + CFLAGS="$CFLAGS -pthread" + LDFLAGS="$LDFLAGS -pthread" ]) ;; QNX-6*) @@ -1854,11 +1783,11 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ SHLIB_SUFFIX=".so" AS_IF([test "$GCC" = yes], [ SHLIB_LD='${CC} -shared' - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' + CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} ], [ SHLIB_LD="/usr/ccs/bin/ld -G -z text" - CC_SEARCH_FLAGS='-R ${LIB_RUNTIME_DIR}' + CC_SEARCH_FLAGS='-R "${LIB_RUNTIME_DIR}"' LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} ]) ;; @@ -1924,7 +1853,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ SHLIB_SUFFIX=".so" AS_IF([test "$GCC" = yes], [ SHLIB_LD='${CC} -shared' - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' + CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} AS_IF([test "$do64bit_ok" = yes], [ AS_IF([test "$arch" = "sparcv9 sparc"], [ @@ -1951,8 +1880,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ *) SHLIB_LD='/usr/ccs/bin/ld -G -z text';; esac - CC_SEARCH_FLAGS='-Wl,-R,${LIB_RUNTIME_DIR}' - LD_SEARCH_FLAGS='-R ${LIB_RUNTIME_DIR}' + CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' + LD_SEARCH_FLAGS='-R "${LIB_RUNTIME_DIR}"' ]) ;; UNIX_SV* | UnixWare-5*) @@ -1965,7 +1894,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ AC_CACHE_CHECK([for ld accepts -Bexport flag], tcl_cv_ld_Bexport, [ hold_ldflags=$LDFLAGS LDFLAGS="$LDFLAGS -Wl,-Bexport" - AC_TRY_LINK(, [int i;], tcl_cv_ld_Bexport=yes, tcl_cv_ld_Bexport=no) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[int i;]])], + [tcl_cv_ld_Bexport=yes],[tcl_cv_ld_Bexport=no]) LDFLAGS=$hold_ldflags]) AS_IF([test $tcl_cv_ld_Bexport = yes], [ LDFLAGS="$LDFLAGS -Wl,-Bexport" @@ -1997,9 +1927,9 @@ dnl # preprocessing tests use only CPPFLAGS. case $system in AIX-*) ;; BSD/OS*) ;; - CYGWIN_*|MINGW32_*) ;; + CYGWIN_*|MINGW32_*|MINGW64_*|MSYS_*) ;; IRIX*) ;; - NetBSD-*|FreeBSD-*|OpenBSD-*) ;; + NetBSD-*|DragonFly-*|FreeBSD-*|OpenBSD-*) ;; Darwin-*) ;; SCO_SV-3.2*) ;; windows) ;; @@ -2021,7 +1951,7 @@ dnl # preprocessing tests use only CPPFLAGS. if test "${GCC}" = "yes" -a ${SHLIB_SUFFIX} = ".dll"; then AC_CACHE_CHECK(for SEH support in compiler, tcl_cv_seh, - AC_TRY_RUN([ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ #define WIN32_LEAN_AND_MEAN #include #undef WIN32_LEAN_AND_MEAN @@ -2036,10 +1966,10 @@ dnl # preprocessing tests use only CPPFLAGS. } return 1; } - ], - tcl_cv_seh=yes, - tcl_cv_seh=no, - tcl_cv_seh=no) + ]])], + [tcl_cv_seh=yes], + [tcl_cv_seh=no], + [tcl_cv_seh=no]) ) if test "$tcl_cv_seh" = "no" ; then AC_DEFINE(HAVE_NO_SEH, 1, @@ -2054,15 +1984,15 @@ dnl # preprocessing tests use only CPPFLAGS. # AC_CACHE_CHECK(for EXCEPTION_DISPOSITION support in include files, tcl_cv_eh_disposition, - AC_TRY_COMPILE([ + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ # define WIN32_LEAN_AND_MEAN # include # undef WIN32_LEAN_AND_MEAN - ],[ + ]], [[ EXCEPTION_DISPOSITION x; - ], - tcl_cv_eh_disposition=yes, - tcl_cv_eh_disposition=no) + ]])], + [tcl_cv_eh_disposition=yes], + [tcl_cv_eh_disposition=no]) ) if test "$tcl_cv_eh_disposition" = "no" ; then AC_DEFINE(EXCEPTION_DISPOSITION, int, @@ -2075,18 +2005,18 @@ dnl # preprocessing tests use only CPPFLAGS. AC_CACHE_CHECK(for winnt.h that ignores VOID define, tcl_cv_winnt_ignore_void, - AC_TRY_COMPILE([ + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #define VOID void #define WIN32_LEAN_AND_MEAN #include #undef WIN32_LEAN_AND_MEAN - ], [ + ]], [[ CHAR c; SHORT s; LONG l; - ], - tcl_cv_winnt_ignore_void=yes, - tcl_cv_winnt_ignore_void=no) + ]])], + [tcl_cv_winnt_ignore_void=yes], + [tcl_cv_winnt_ignore_void=no]) ) if test "$tcl_cv_winnt_ignore_void" = "yes" ; then AC_DEFINE(HAVE_WINNT_IGNORE_VOID, 1, @@ -2100,22 +2030,25 @@ dnl # preprocessing tests use only CPPFLAGS. AC_CACHE_CHECK(for cast to union support, tcl_cv_cast_to_union, - AC_TRY_COMPILE([], - [ + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ union foo { int i; double d; }; union foo f = (union foo) (int) 0; - ], - tcl_cv_cast_to_union=yes, - tcl_cv_cast_to_union=no) + ]])], + [tcl_cv_cast_to_union=yes], + [tcl_cv_cast_to_union=no]) ) if test "$tcl_cv_cast_to_union" = "yes"; then AC_DEFINE(HAVE_CAST_TO_UNION, 1, [Defined when compiler supports casting to union type.]) fi + AC_CHECK_HEADER(stdbool.h, [AC_DEFINE(HAVE_STDBOOL_H, 1, [Do we have ?])],) + AC_SUBST(CFLAGS_DEBUG) AC_SUBST(CFLAGS_OPTIMIZE) AC_SUBST(CFLAGS_WARNING) + AC_SUBST(LDFLAGS_DEBUG) + AC_SUBST(LDFLAGS_OPTIMIZE) AC_SUBST(STLIB_LD) AC_SUBST(SHLIB_LD) @@ -2155,7 +2088,7 @@ dnl # preprocessing tests use only CPPFLAGS. AC_DEFUN([TEA_SERIAL_PORT], [ AC_CHECK_HEADERS(sys/modem.h) AC_CACHE_CHECK([termios vs. termio vs. sgtty], tcl_cv_api_serial, [ - AC_TRY_RUN([ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include int main() { @@ -2166,9 +2099,9 @@ int main() { return 0; } return 1; -}], tcl_cv_api_serial=termios, tcl_cv_api_serial=no, tcl_cv_api_serial=no) +}]])],[tcl_cv_api_serial=termios],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) if test $tcl_cv_api_serial = no ; then - AC_TRY_RUN([ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include int main() { @@ -2178,10 +2111,10 @@ int main() { return 0; } return 1; -}], tcl_cv_api_serial=termio, tcl_cv_api_serial=no, tcl_cv_api_serial=no) +}]])],[tcl_cv_api_serial=termio],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) fi if test $tcl_cv_api_serial = no ; then - AC_TRY_RUN([ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include int main() { @@ -2192,10 +2125,10 @@ int main() { return 0; } return 1; -}], tcl_cv_api_serial=sgtty, tcl_cv_api_serial=no, tcl_cv_api_serial=no) +}]])],[tcl_cv_api_serial=sgtty],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) fi if test $tcl_cv_api_serial = no ; then - AC_TRY_RUN([ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include @@ -2208,10 +2141,10 @@ int main() { return 0; } return 1; -}], tcl_cv_api_serial=termios, tcl_cv_api_serial=no, tcl_cv_api_serial=no) +}]])],[tcl_cv_api_serial=termios],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) fi if test $tcl_cv_api_serial = no; then - AC_TRY_RUN([ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include @@ -2223,10 +2156,10 @@ int main() { return 0; } return 1; - }], tcl_cv_api_serial=termio, tcl_cv_api_serial=no, tcl_cv_api_serial=no) + }]])],[tcl_cv_api_serial=termio],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) fi if test $tcl_cv_api_serial = no; then - AC_TRY_RUN([ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include @@ -2239,7 +2172,7 @@ int main() { return 0; } return 1; -}], tcl_cv_api_serial=sgtty, tcl_cv_api_serial=none, tcl_cv_api_serial=none) +}]])],[tcl_cv_api_serial=sgtty],[tcl_cv_api_serial=none],[tcl_cv_api_serial=none]) fi]) case $tcl_cv_api_serial in termios) AC_DEFINE(USE_TERMIOS, 1, [Use the termios API for serial lines]);; @@ -2248,97 +2181,6 @@ int main() { esac ]) -#-------------------------------------------------------------------- -# TEA_MISSING_POSIX_HEADERS -# -# Supply substitutes for missing POSIX header files. Special -# notes: -# - stdlib.h doesn't define strtol, strtoul, or -# strtod in some versions of SunOS -# - some versions of string.h don't declare procedures such -# as strstr -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# NO_DIRENT_H -# NO_ERRNO_H -# NO_VALUES_H -# HAVE_LIMITS_H or NO_LIMITS_H -# NO_STDLIB_H -# NO_STRING_H -# NO_SYS_WAIT_H -# NO_DLFCN_H -# HAVE_SYS_PARAM_H -# -# HAVE_STRING_H ? -# -# tkUnixPort.h checks for HAVE_LIMITS_H, so do both HAVE and -# CHECK on limits.h -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_MISSING_POSIX_HEADERS], [ - AC_CACHE_CHECK([dirent.h], tcl_cv_dirent_h, [ - AC_TRY_LINK([#include -#include ], [ -#ifndef _POSIX_SOURCE -# ifdef __Lynx__ - /* - * Generate compilation error to make the test fail: Lynx headers - * are only valid if really in the POSIX environment. - */ - - missing_procedure(); -# endif -#endif -DIR *d; -struct dirent *entryPtr; -char *p; -d = opendir("foobar"); -entryPtr = readdir(d); -p = entryPtr->d_name; -closedir(d); -], tcl_cv_dirent_h=yes, tcl_cv_dirent_h=no)]) - - if test $tcl_cv_dirent_h = no; then - AC_DEFINE(NO_DIRENT_H, 1, [Do we have ?]) - fi - - # TEA specific: - AC_CHECK_HEADER(errno.h, , [AC_DEFINE(NO_ERRNO_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(float.h, , [AC_DEFINE(NO_FLOAT_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(values.h, , [AC_DEFINE(NO_VALUES_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(limits.h, - [AC_DEFINE(HAVE_LIMITS_H, 1, [Do we have ?])], - [AC_DEFINE(NO_LIMITS_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(stdlib.h, tcl_ok=1, tcl_ok=0) - AC_EGREP_HEADER(strtol, stdlib.h, , tcl_ok=0) - AC_EGREP_HEADER(strtoul, stdlib.h, , tcl_ok=0) - AC_EGREP_HEADER(strtod, stdlib.h, , tcl_ok=0) - if test $tcl_ok = 0; then - AC_DEFINE(NO_STDLIB_H, 1, [Do we have ?]) - fi - AC_CHECK_HEADER(string.h, tcl_ok=1, tcl_ok=0) - AC_EGREP_HEADER(strstr, string.h, , tcl_ok=0) - AC_EGREP_HEADER(strerror, string.h, , tcl_ok=0) - - # See also memmove check below for a place where NO_STRING_H can be - # set and why. - - if test $tcl_ok = 0; then - AC_DEFINE(NO_STRING_H, 1, [Do we have ?]) - fi - - AC_CHECK_HEADER(sys/wait.h, , [AC_DEFINE(NO_SYS_WAIT_H, 1, [Do we have ?])]) - AC_CHECK_HEADER(dlfcn.h, , [AC_DEFINE(NO_DLFCN_H, 1, [Do we have ?])]) - - # OS/390 lacks sys/param.h (and doesn't need it, by chance). - AC_HAVE_HEADERS(sys/param.h) -]) - #-------------------------------------------------------------------- # TEA_PATH_X # @@ -2374,7 +2216,7 @@ AC_DEFUN([TEA_PATH_UNIX_X], [ not_really_there="" if test "$no_x" = ""; then if test "$x_includes" = ""; then - AC_TRY_CPP([#include ], , not_really_there="yes") + AC_PREPROC_IFELSE([AC_LANG_SOURCE([[#include ]])],[],[not_really_there="yes"]) else if test ! -r $x_includes/X11/Xlib.h; then not_really_there="yes" @@ -2384,7 +2226,7 @@ AC_DEFUN([TEA_PATH_UNIX_X], [ if test "$no_x" = "yes" -o "$not_really_there" = "yes"; then AC_MSG_CHECKING([for X11 header files]) found_xincludes="no" - AC_TRY_CPP([#include ], found_xincludes="yes", found_xincludes="no") + AC_PREPROC_IFELSE([AC_LANG_SOURCE([[#include ]])],[found_xincludes="yes"],[found_xincludes="no"]) if test "$found_xincludes" = "no"; then dirs="/usr/unsupported/include /usr/local/include /usr/X386/include /usr/X11R6/include /usr/X11R5/include /usr/include/X11R5 /usr/include/X11R4 /usr/openwin/include /usr/X11/include /usr/sww/include" for i in $dirs ; do @@ -2490,6 +2332,7 @@ AC_DEFUN([TEA_BLOCKING_STYLE], [ # HAVE_TM_GMTOFF # HAVE_TM_TZADJ # HAVE_TIMEZONE_VAR +# #-------------------------------------------------------------------- AC_DEFUN([TEA_TIME_HANDLER], [ @@ -2497,18 +2340,20 @@ AC_DEFUN([TEA_TIME_HANDLER], [ AC_HEADER_TIME AC_STRUCT_TIMEZONE - AC_CHECK_FUNCS(gmtime_r localtime_r) + AC_CHECK_FUNCS(gmtime_r localtime_r mktime) AC_CACHE_CHECK([tm_tzadj in struct tm], tcl_cv_member_tm_tzadj, [ - AC_TRY_COMPILE([#include ], [struct tm tm; tm.tm_tzadj;], - tcl_cv_member_tm_tzadj=yes, tcl_cv_member_tm_tzadj=no)]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[struct tm tm; (void)tm.tm_tzadj;]])], + [tcl_cv_member_tm_tzadj=yes], + [tcl_cv_member_tm_tzadj=no])]) if test $tcl_cv_member_tm_tzadj = yes ; then AC_DEFINE(HAVE_TM_TZADJ, 1, [Should we use the tm_tzadj field of struct tm?]) fi AC_CACHE_CHECK([tm_gmtoff in struct tm], tcl_cv_member_tm_gmtoff, [ - AC_TRY_COMPILE([#include ], [struct tm tm; tm.tm_gmtoff;], - tcl_cv_member_tm_gmtoff=yes, tcl_cv_member_tm_gmtoff=no)]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[struct tm tm; (void)tm.tm_gmtoff;]])], + [tcl_cv_member_tm_gmtoff=yes], + [tcl_cv_member_tm_gmtoff=no])]) if test $tcl_cv_member_tm_gmtoff = yes ; then AC_DEFINE(HAVE_TM_GMTOFF, 1, [Should we use the tm_gmtoff field of struct tm?]) fi @@ -2518,11 +2363,12 @@ AC_DEFUN([TEA_TIME_HANDLER], [ # (like convex) have timezone functions, etc. # AC_CACHE_CHECK([long timezone variable], tcl_cv_timezone_long, [ - AC_TRY_COMPILE([#include ], - [extern long timezone; + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include +#include ]], + [[extern long timezone; timezone += 1; - exit (0);], - tcl_cv_timezone_long=yes, tcl_cv_timezone_long=no)]) + exit (0);]])], + [tcl_cv_timezone_long=yes], [tcl_cv_timezone_long=no])]) if test $tcl_cv_timezone_long = yes ; then AC_DEFINE(HAVE_TIMEZONE_VAR, 1, [Should we use the global timezone variable?]) else @@ -2530,11 +2376,12 @@ AC_DEFUN([TEA_TIME_HANDLER], [ # On some systems (eg IRIX 6.2), timezone is a time_t and not a long. # AC_CACHE_CHECK([time_t timezone variable], tcl_cv_timezone_time, [ - AC_TRY_COMPILE([#include ], - [extern time_t timezone; + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include +#include ]], + [[extern time_t timezone; timezone += 1; - exit (0);], - tcl_cv_timezone_time=yes, tcl_cv_timezone_time=no)]) + exit (0);]])], + [tcl_cv_timezone_time=yes], [tcl_cv_timezone_time=no])]) if test $tcl_cv_timezone_time = yes ; then AC_DEFINE(HAVE_TIMEZONE_VAR, 1, [Should we use the global timezone variable?]) fi @@ -2564,7 +2411,8 @@ AC_DEFUN([TEA_BUGGY_STRTOD], [ AC_CHECK_FUNC(strtod, tcl_strtod=1, tcl_strtod=0) if test "$tcl_strtod" = 1; then AC_CACHE_CHECK([for Solaris2.4/Tru64 strtod bugs], tcl_cv_strtod_buggy,[ - AC_TRY_RUN([ + AC_RUN_IFELSE([AC_LANG_SOURCE([[ + #include extern double strtod(); int main() { char *infString="Inf", *nanString="NaN", *spaceString=" "; @@ -2583,8 +2431,8 @@ AC_DEFUN([TEA_BUGGY_STRTOD], [ exit(1); } exit(0); - }], tcl_cv_strtod_buggy=ok, tcl_cv_strtod_buggy=buggy, - tcl_cv_strtod_buggy=buggy)]) + }]])], [tcl_cv_strtod_buggy=ok], [tcl_cv_strtod_buggy=buggy], + [tcl_cv_strtod_buggy=buggy])]) if test "$tcl_cv_strtod_buggy" = buggy; then AC_LIBOBJ([fixstrtod]) USE_COMPAT=1 @@ -2597,38 +2445,30 @@ AC_DEFUN([TEA_BUGGY_STRTOD], [ # TEA_TCL_LINK_LIBS # # Search for the libraries needed to link the Tcl shell. -# Things like the math library (-lm) and socket stuff (-lsocket vs. -# -lnsl) are dealt with here. +# Things like the math library (-lm), socket stuff (-lsocket vs. +# -lnsl), zlib (-lz) and libtommath (-ltommath) are dealt with here. # # Arguments: -# Requires the following vars to be set in the Makefile: -# DL_LIBS (not in TEA, only needed in core) -# LIBS -# MATH_LIBS +# None. # # Results: # -# Substitutes the following vars: -# TCL_LIBS -# MATH_LIBS -# # Might append to the following vars: # LIBS +# MATH_LIBS # # Might define the following vars: # HAVE_NET_ERRNO_H +# #-------------------------------------------------------------------- AC_DEFUN([TEA_TCL_LINK_LIBS], [ #-------------------------------------------------------------------- # On a few very rare systems, all of the libm.a stuff is # already in libc.a. Set compiler flags accordingly. - # Also, Linux requires the "ieee" library for math to work - # right (and it must appear before "-lm"). #-------------------------------------------------------------------- AC_CHECK_FUNC(sin, MATH_LIBS="", MATH_LIBS="-lm") - AC_CHECK_LIB(ieee, main, [MATH_LIBS="-lieee $MATH_LIBS"]) #-------------------------------------------------------------------- # Interactive UNIX requires -linet instead of -lsocket, plus it @@ -2670,13 +2510,10 @@ AC_DEFUN([TEA_TCL_LINK_LIBS], [ fi AC_CHECK_FUNC(gethostbyname, , [AC_CHECK_LIB(nsl, gethostbyname, [LIBS="$LIBS -lnsl"])]) - - # TEA specific: Don't perform the eval of the libraries here because - # DL_LIBS won't be set until we call TEA_CONFIG_CFLAGS - - TCL_LIBS='${DL_LIBS} ${LIBS} ${MATH_LIBS}' - AC_SUBST(TCL_LIBS) - AC_SUBST(MATH_LIBS) + AC_CHECK_FUNC(mp_log_u32, , [AC_CHECK_LIB(tommath, mp_log_u32, + [LIBS="$LIBS -ltommath"])]) + AC_CHECK_FUNC(deflateSetHeader, , [AC_CHECK_LIB(z, deflateSetHeader, + [LIBS="$LIBS -lz"])]) ]) #-------------------------------------------------------------------- @@ -2694,15 +2531,16 @@ AC_DEFUN([TEA_TCL_LINK_LIBS], [ # _ISOC99_SOURCE # _LARGEFILE64_SOURCE # _LARGEFILE_SOURCE64 +# #-------------------------------------------------------------------- AC_DEFUN([TEA_TCL_EARLY_FLAG],[ AC_CACHE_VAL([tcl_cv_flag_]translit($1,[A-Z],[a-z]), - AC_TRY_COMPILE([$2], $3, [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no, - AC_TRY_COMPILE([[#define ]$1[ 1 -]$2], $3, - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=yes, - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no))) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$2]], [[$3]])], + [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no,[AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[[#define ]$1[ 1 +]$2]], [[$3]])], + [tcl_cv_flag_]translit($1,[A-Z],[a-z])=yes, + [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no)])) if test ["x${tcl_cv_flag_]translit($1,[A-Z],[a-z])[}" = "xyes"] ; then AC_DEFINE($1, 1, [Add the ]$1[ flag when building]) tcl_flags="$tcl_flags $1" @@ -2738,9 +2576,10 @@ AC_DEFUN([TEA_TCL_EARLY_FLAGS],[ # Might define the following vars: # TCL_WIDE_INT_IS_LONG # TCL_WIDE_INT_TYPE -# HAVE_STRUCT_DIRENT64 +# HAVE_STRUCT_DIRENT64, HAVE_DIR64 # HAVE_STRUCT_STAT64 # HAVE_TYPE_OFF64_T +# #-------------------------------------------------------------------- AC_DEFUN([TEA_TCL_64BIT_FLAGS], [ @@ -2748,17 +2587,17 @@ AC_DEFUN([TEA_TCL_64BIT_FLAGS], [ AC_CACHE_VAL(tcl_cv_type_64bit,[ tcl_cv_type_64bit=none # See if the compiler knows natively about __int64 - AC_TRY_COMPILE(,[__int64 value = (__int64) 0;], - tcl_type_64bit=__int64, tcl_type_64bit="long long") - # See if we should use long anyway Note that we substitute in the + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[__int64 value = (__int64) 0;]])], + [tcl_type_64bit=__int64],[tcl_type_64bit="long long"]) + # See if we could use long anyway Note that we substitute in the # type that is our current guess for a 64-bit type inside this check # program, so it should be modified only carefully... - AC_TRY_COMPILE(,[switch (0) { - case 1: case (sizeof(]${tcl_type_64bit}[)==sizeof(long)): ; - }],tcl_cv_type_64bit=${tcl_type_64bit})]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[switch (0) { + case 1: case (sizeof(${tcl_type_64bit})==sizeof(long)): ; + }]])],[tcl_cv_type_64bit=${tcl_type_64bit}],[])]) if test "${tcl_cv_type_64bit}" = none ; then - AC_DEFINE(TCL_WIDE_INT_IS_LONG, 1, [Are wide integers to be implemented with C 'long's?]) - AC_MSG_RESULT([using long]) + AC_DEFINE(TCL_WIDE_INT_IS_LONG, 1, [Do 'long' and 'long long' have the same size (64-bit)?]) + AC_MSG_RESULT([yes]) elif test "${tcl_cv_type_64bit}" = "__int64" \ -a "${TEA_PLATFORM}" = "windows" ; then # TEA specific: We actually want to use the default tcl.h checks in @@ -2771,17 +2610,26 @@ AC_DEFUN([TEA_TCL_64BIT_FLAGS], [ # Now check for auxiliary declarations AC_CACHE_CHECK([for struct dirent64], tcl_cv_struct_dirent64,[ - AC_TRY_COMPILE([#include -#include ],[struct dirent64 p;], - tcl_cv_struct_dirent64=yes,tcl_cv_struct_dirent64=no)]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include +#include ]], [[struct dirent64 p;]])], + [tcl_cv_struct_dirent64=yes],[tcl_cv_struct_dirent64=no])]) if test "x${tcl_cv_struct_dirent64}" = "xyes" ; then AC_DEFINE(HAVE_STRUCT_DIRENT64, 1, [Is 'struct dirent64' in ?]) fi + AC_CACHE_CHECK([for DIR64], tcl_cv_DIR64,[ + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include +#include ]], [[struct dirent64 *p; DIR64 d = opendir64("."); + p = readdir64(d); rewinddir64(d); closedir64(d);]])], + [tcl_cv_DIR64=yes], [tcl_cv_DIR64=no])]) + if test "x${tcl_cv_DIR64}" = "xyes" ; then + AC_DEFINE(HAVE_DIR64, 1, [Is 'DIR64' in ?]) + fi + AC_CACHE_CHECK([for struct stat64], tcl_cv_struct_stat64,[ - AC_TRY_COMPILE([#include ],[struct stat64 p; -], - tcl_cv_struct_stat64=yes,tcl_cv_struct_stat64=no)]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[struct stat64 p; +]])], + [tcl_cv_struct_stat64=yes], [tcl_cv_struct_stat64=no])]) if test "x${tcl_cv_struct_stat64}" = "xyes" ; then AC_DEFINE(HAVE_STRUCT_STAT64, 1, [Is 'struct stat64' in ?]) fi @@ -2789,9 +2637,9 @@ AC_DEFUN([TEA_TCL_64BIT_FLAGS], [ AC_CHECK_FUNCS(open64 lseek64) AC_MSG_CHECKING([for off64_t]) AC_CACHE_VAL(tcl_cv_type_off64_t,[ - AC_TRY_COMPILE([#include ],[off64_t offset; -], - tcl_cv_type_off64_t=yes,tcl_cv_type_off64_t=no)]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[off64_t offset; +]])], + [tcl_cv_type_off64_t=yes], [tcl_cv_type_off64_t=no])]) dnl Define HAVE_TYPE_OFF64_T only when the off64_t type and the dnl functions lseek64 and open64 are defined. if test "x${tcl_cv_type_off64_t}" = "xyes" && \ @@ -2840,23 +2688,14 @@ AC_DEFUN([TEA_TCL_64BIT_FLAGS], [ #------------------------------------------------------------------------ AC_DEFUN([TEA_INIT], [ - # TEA extensions pass this us the version of TEA they think they - # are compatible with. - TEA_VERSION="3.9" + TEA_VERSION="3.13" - AC_MSG_CHECKING([for correct TEA configuration]) + AC_MSG_CHECKING([TEA configuration]) if test x"${PACKAGE_NAME}" = x ; then AC_MSG_ERROR([ -The PACKAGE_NAME variable must be defined by your TEA configure.in]) - fi - if test x"$1" = x ; then - AC_MSG_ERROR([ -TEA version not specified.]) - elif test "$1" != "${TEA_VERSION}" ; then - AC_MSG_RESULT([warning: requested TEA version "$1", have "${TEA_VERSION}"]) - else - AC_MSG_RESULT([ok (TEA ${TEA_VERSION})]) +The PACKAGE_NAME variable must be defined by your TEA configure.ac]) fi + AC_MSG_RESULT([ok (TEA ${TEA_VERSION})]) # If the user did not set CFLAGS, set it now to keep macros # like AC_PROG_CC and AC_TRY_COMPILE from adding "-g -O2". @@ -2865,15 +2704,14 @@ TEA version not specified.]) fi case "`uname -s`" in - *win32*|*WIN32*|*MINGW32_*) - AC_CHECK_PROG(CYGPATH, cygpath, cygpath -w, echo) + *win32*|*WIN32*|*MINGW32_*|*MINGW64_*|*MSYS_*) + AC_CHECK_PROG(CYGPATH, cygpath, cygpath -m, echo) EXEEXT=".exe" TEA_PLATFORM="windows" ;; *CYGWIN_*) - CYGPATH=echo EXEEXT=".exe" - # TEA_PLATFORM is determined later in LOAD_TCLCONFIG + # CYGPATH and TEA_PLATFORM are determined later in LOAD_TCLCONFIG ;; *) CYGPATH=echo @@ -2907,6 +2745,8 @@ TEA version not specified.]) # This package name must be replaced statically for AC_SUBST to work AC_SUBST(PKG_LIB_FILE) + AC_SUBST(PKG_LIB_FILE8) + AC_SUBST(PKG_LIB_FILE9) # Substitute STUB_LIB_FILE in case package creates a stub library too. AC_SUBST(PKG_STUB_LIB_FILE) @@ -2919,6 +2759,9 @@ TEA version not specified.]) AC_SUBST(PKG_INCLUDES) AC_SUBST(PKG_LIBS) AC_SUBST(PKG_CFLAGS) + + # Configure the installer. + TEA_INSTALLER ]) #------------------------------------------------------------------------ @@ -3111,7 +2954,7 @@ AC_DEFUN([TEA_ADD_LIBS], [ for i in $vars; do if test "${TEA_PLATFORM}" = "windows" -a "$GCC" = "yes" ; then # Convert foo.lib to -lfoo for GCC. No-op if not *.lib - i=`echo "$i" | sed -e 's/^\([[^-]].*\)\.lib[$]/-l\1/i'` + i=`echo "$i" | sed -e 's/^\([[^-]].*\)\.[[lL]][[iI]][[bB]][$]/-l\1/'` fi PKG_LIBS="$PKG_LIBS $i" done @@ -3194,7 +3037,7 @@ AC_DEFUN([TEA_PREFIX], [ # TEA_SETUP_COMPILER_CC -- # # Do compiler checks the way we want. This is just a replacement -# for AC_PROG_CC in TEA configure.in files to make them cleaner. +# for AC_PROG_CC in TEA configure.ac files to make them cleaner. # # Arguments: # none @@ -3210,15 +3053,6 @@ AC_DEFUN([TEA_SETUP_COMPILER_CC], [ AC_PROG_CC AC_PROG_CPP - INSTALL="\$(SHELL) \$(srcdir)/tclconfig/install-sh -c" - AC_SUBST(INSTALL) - INSTALL_DATA="\${INSTALL} -m 644" - AC_SUBST(INSTALL_DATA) - INSTALL_PROGRAM="\${INSTALL}" - AC_SUBST(INSTALL_PROGRAM) - INSTALL_SCRIPT="\${INSTALL}" - AC_SUBST(INSTALL_SCRIPT) - #-------------------------------------------------------------------- # Checks to see if the make program sets the $MAKE variable. #-------------------------------------------------------------------- @@ -3265,7 +3099,7 @@ AC_DEFUN([TEA_SETUP_COMPILER], [ AC_CACHE_CHECK([if the compiler understands -pipe], tcl_cv_cc_pipe, [ hold_cflags=$CFLAGS; CFLAGS="$CFLAGS -pipe" - AC_TRY_COMPILE(,, tcl_cv_cc_pipe=yes, tcl_cv_cc_pipe=no) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])],[tcl_cv_cc_pipe=yes],[tcl_cv_cc_pipe=no]) CFLAGS=$hold_cflags]) if test $tcl_cv_cc_pipe = yes; then CFLAGS="$CFLAGS -pipe" @@ -3277,14 +3111,6 @@ AC_DEFUN([TEA_SETUP_COMPILER], [ #-------------------------------------------------------------------- AC_C_BIGENDIAN - if test "${TEA_PLATFORM}" = "unix" ; then - TEA_TCL_LINK_LIBS - TEA_MISSING_POSIX_HEADERS - # Let the user call this, because if it triggers, they will - # need a compat/strtod.c that is correct. Users can also - # use Tcl_GetDouble(FromObj) instead. - #TEA_BUGGY_STRTOD - fi ]) #------------------------------------------------------------------------ @@ -3315,7 +3141,7 @@ AC_DEFUN([TEA_SETUP_COMPILER], [ AC_DEFUN([TEA_MAKE_LIB], [ if test "${TEA_PLATFORM}" = "windows" -a "$GCC" != "yes"; then MAKE_STATIC_LIB="\${STLIB_LD} -out:\[$]@ \$(PKG_OBJECTS)" - MAKE_SHARED_LIB="\${SHLIB_LD} \${SHLIB_LD_LIBS} \${LDFLAGS_DEFAULT} -out:\[$]@ \$(PKG_OBJECTS)" + MAKE_SHARED_LIB="\${SHLIB_LD} \${LDFLAGS} \${LDFLAGS_DEFAULT} -out:\[$]@ \$(PKG_OBJECTS) \${SHLIB_LD_LIBS}" AC_EGREP_CPP([manifest needed], [ #if defined(_MSC_VER) && _MSC_VER >= 1400 print("manifest needed") @@ -3330,7 +3156,7 @@ print("manifest needed") MAKE_STUB_LIB="\${STLIB_LD} -nodefaultlib -out:\[$]@ \$(PKG_STUB_OBJECTS)" else MAKE_STATIC_LIB="\${STLIB_LD} \[$]@ \$(PKG_OBJECTS)" - MAKE_SHARED_LIB="\${SHLIB_LD} -o \[$]@ \$(PKG_OBJECTS) \${SHLIB_LD_LIBS}" + MAKE_SHARED_LIB="\${SHLIB_LD} \${LDFLAGS} \${LDFLAGS_DEFAULT} -o \[$]@ \$(PKG_OBJECTS) \${SHLIB_LD_LIBS}" MAKE_STUB_LIB="\${STLIB_LD} \[$]@ \$(PKG_STUB_OBJECTS)" fi @@ -3346,6 +3172,13 @@ print("manifest needed") # substituted. (@@@ Might not be necessary anymore) #-------------------------------------------------------------------- + PACKAGE_LIB_PREFIX8="${PACKAGE_LIB_PREFIX}" + PACKAGE_LIB_PREFIX9="${PACKAGE_LIB_PREFIX}tcl9" + if test "${TCL_MAJOR_VERSION}" -gt 8 ; then + PACKAGE_LIB_PREFIX="${PACKAGE_LIB_PREFIX9}" + else + PACKAGE_LIB_PREFIX="${PACKAGE_LIB_PREFIX8}" + fi if test "${TEA_PLATFORM}" = "windows" ; then if test "${SHARED_BUILD}" = "1" ; then # We force the unresolved linking of symbols that are really in @@ -3357,15 +3190,19 @@ print("manifest needed") if test "$GCC" = "yes"; then SHLIB_LD_LIBS="${SHLIB_LD_LIBS} -static-libgcc" fi - eval eval "PKG_LIB_FILE=${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE8=${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE9=${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE=${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" else - eval eval "PKG_LIB_FILE=${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" if test "$GCC" = "yes"; then - PKG_LIB_FILE=lib${PKG_LIB_FILE} + PACKAGE_LIB_PREFIX=lib${PACKAGE_LIB_PREFIX} fi + eval eval "PKG_LIB_FILE8=${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE9=${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE=${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" fi # Some packages build their own stubs libraries - eval eval "PKG_STUB_LIB_FILE=${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}" + eval eval "PKG_STUB_LIB_FILE=${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}" if test "$GCC" = "yes"; then PKG_STUB_LIB_FILE=lib${PKG_STUB_LIB_FILE} fi @@ -3379,13 +3216,17 @@ print("manifest needed") if test x"${TK_BIN_DIR}" != x ; then SHLIB_LD_LIBS="${SHLIB_LD_LIBS} ${TK_STUB_LIB_SPEC}" fi - eval eval "PKG_LIB_FILE=lib${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE8=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE9=lib${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" RANLIB=: else - eval eval "PKG_LIB_FILE=lib${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" + eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" fi # Some packages build their own stubs libraries - eval eval "PKG_STUB_LIB_FILE=lib${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}" + eval eval "PKG_STUB_LIB_FILE=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}" fi # These are escaped so that only CFLAGS is picked up at configure time. @@ -3930,6 +3771,7 @@ AC_DEFUN([TEA_PATH_CONFIG], [ `ls -d ${prefix}/lib 2>/dev/null` \ `ls -d /usr/local/lib 2>/dev/null` \ `ls -d /usr/contrib/lib 2>/dev/null` \ + `ls -d /usr/pkg/lib 2>/dev/null` \ `ls -d /usr/lib 2>/dev/null` \ `ls -d /usr/lib64 2>/dev/null` \ ; do @@ -4071,18 +3913,18 @@ AC_DEFUN([TEA_EXPORT_CONFIG], [ # pkglibdir must be a fully qualified path and (not ${exec_prefix}/lib) eval pkglibdir="[$]{libdir}/$1${PACKAGE_VERSION}" if test "${TCL_LIB_VERSIONS_OK}" = "ok"; then - eval $1_LIB_FLAG="-l$1${PACKAGE_VERSION}${DBGX}" - eval $1_STUB_LIB_FLAG="-l$1stub${PACKAGE_VERSION}${DBGX}" + eval $1_LIB_FLAG="-l$1${PACKAGE_VERSION}" + eval $1_STUB_LIB_FLAG="-l$1stub${PACKAGE_VERSION}" else - eval $1_LIB_FLAG="-l$1`echo ${PACKAGE_VERSION} | tr -d .`${DBGX}" - eval $1_STUB_LIB_FLAG="-l$1stub`echo ${PACKAGE_VERSION} | tr -d .`${DBGX}" + eval $1_LIB_FLAG="-l$1`echo ${PACKAGE_VERSION} | tr -d .`" + eval $1_STUB_LIB_FLAG="-l$1stub`echo ${PACKAGE_VERSION} | tr -d .`" fi - $1_BUILD_LIB_SPEC="-L`pwd` ${$1_LIB_FLAG}" - $1_LIB_SPEC="-L${pkglibdir} ${$1_LIB_FLAG}" - $1_BUILD_STUB_LIB_SPEC="-L`pwd` [$]{$1_STUB_LIB_FLAG}" - $1_STUB_LIB_SPEC="-L${pkglibdir} [$]{$1_STUB_LIB_FLAG}" - $1_BUILD_STUB_LIB_PATH="`pwd`/[$]{PKG_STUB_LIB_FILE}" - $1_STUB_LIB_PATH="${pkglibdir}/[$]{PKG_STUB_LIB_FILE}" + $1_BUILD_LIB_SPEC="-L`$CYGPATH $(pwd)` ${$1_LIB_FLAG}" + $1_LIB_SPEC="-L`$CYGPATH ${pkglibdir}` ${$1_LIB_FLAG}" + $1_BUILD_STUB_LIB_SPEC="-L`$CYGPATH $(pwd)` [$]{$1_STUB_LIB_FLAG}" + $1_STUB_LIB_SPEC="-L`$CYGPATH ${pkglibdir}` [$]{$1_STUB_LIB_FLAG}" + $1_BUILD_STUB_LIB_PATH="`$CYGPATH $(pwd)`/[$]{PKG_STUB_LIB_FILE}" + $1_STUB_LIB_PATH="`$CYGPATH ${pkglibdir}`/[$]{PKG_STUB_LIB_FILE}" AC_SUBST($1_BUILD_LIB_SPEC) AC_SUBST($1_LIB_SPEC) @@ -4098,71 +3940,128 @@ AC_DEFUN([TEA_EXPORT_CONFIG], [ #------------------------------------------------------------------------ -# TEA_PATH_CELIB -- +# TEA_INSTALLER -- # -# Locate Keuchel's celib emulation layer for targeting Win/CE +# Configure the installer. # # Arguments: # none # # Results: -# -# Adds the following arguments to configure: -# --with-celib=... -# -# Defines the following vars: -# CELIB_DIR Full path to the directory containing -# the include and platform lib files +# Substitutes the following vars: +# INSTALL +# INSTALL_DATA_DIR +# INSTALL_DATA +# INSTALL_PROGRAM +# INSTALL_SCRIPT +# INSTALL_LIBRARY #------------------------------------------------------------------------ -AC_DEFUN([TEA_PATH_CELIB], [ - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-celib +AC_DEFUN([TEA_INSTALLER], [ + INSTALL='$(SHELL) $(srcdir)/tclconfig/install-sh -c' + INSTALL_DATA_DIR='${INSTALL} -d -m 755' + INSTALL_DATA='${INSTALL} -m 644' + INSTALL_PROGRAM='${INSTALL} -m 755' + INSTALL_SCRIPT='${INSTALL} -m 755' - if test x"${no_celib}" = x ; then - # we reset no_celib in case something fails here - no_celib=true - AC_ARG_WITH(celib,[ --with-celib=DIR use Windows/CE support library from DIR], with_celibconfig=${withval}) - AC_MSG_CHECKING([for Windows/CE celib directory]) - AC_CACHE_VAL(ac_cv_c_celibconfig,[ - # First check to see if --with-celibconfig was specified. - if test x"${with_celibconfig}" != x ; then - if test -d "${with_celibconfig}/inc" ; then - ac_cv_c_celibconfig=`(cd ${with_celibconfig}; pwd)` - else - AC_MSG_ERROR([${with_celibconfig} directory doesn't contain inc directory]) - fi - fi + TEA_CONFIG_SYSTEM + case $system in + HP-UX-*) INSTALL_LIBRARY='${INSTALL} -m 755' ;; + *) INSTALL_LIBRARY='${INSTALL} -m 644' ;; + esac - # then check for a celib library - if test x"${ac_cv_c_celibconfig}" = x ; then - for i in \ - ../celib-palm-3.0 \ - ../celib \ - ../../celib-palm-3.0 \ - ../../celib \ - `ls -dr ../celib-*3.[[0-9]]* 2>/dev/null` \ - ${srcdir}/../celib-palm-3.0 \ - ${srcdir}/../celib \ - `ls -dr ${srcdir}/../celib-*3.[[0-9]]* 2>/dev/null` \ - ; do - if test -d "$i/inc" ; then - ac_cv_c_celibconfig=`(cd $i; pwd)` - break - fi - done - fi - ]) - if test x"${ac_cv_c_celibconfig}" = x ; then - AC_MSG_ERROR([Cannot find celib support library directory]) - else - no_celib= - CELIB_DIR=${ac_cv_c_celibconfig} - CELIB_DIR=`echo "$CELIB_DIR" | sed -e 's!\\\!/!g'` - AC_MSG_RESULT([found $CELIB_DIR]) - fi - fi + AC_SUBST(INSTALL) + AC_SUBST(INSTALL_DATA_DIR) + AC_SUBST(INSTALL_DATA) + AC_SUBST(INSTALL_PROGRAM) + AC_SUBST(INSTALL_SCRIPT) + AC_SUBST(INSTALL_LIBRARY) ]) + +### +# Tip 430 - ZipFS Modifications +### +#------------------------------------------------------------------------ +# TEA_ZIPFS_SUPPORT +# Locate a zip encoder installed on the system path, or none. +# +# Arguments: +# none +# +# Results: +# Substitutes the following vars: +# MACHER_PROG +# ZIP_PROG +# ZIP_PROG_OPTIONS +# ZIP_PROG_VFSSEARCH +# ZIP_INSTALL_OBJS +#------------------------------------------------------------------------ + +AC_DEFUN([TEA_ZIPFS_SUPPORT], [ + MACHER_PROG="" + ZIP_PROG="" + ZIP_PROG_OPTIONS="" + ZIP_PROG_VFSSEARCH="" + ZIP_INSTALL_OBJS="" + + AC_MSG_CHECKING([for macher]) + AC_CACHE_VAL(ac_cv_path_macher, [ + search_path=`echo ${PATH} | sed -e 's/:/ /g'` + for dir in $search_path ; do + for j in `ls -r $dir/macher 2> /dev/null` \ + `ls -r $dir/macher 2> /dev/null` ; do + if test x"$ac_cv_path_macher" = x ; then + if test -f "$j" ; then + ac_cv_path_macher=$j + break + fi + fi + done + done + ]) + if test -f "$ac_cv_path_macher" ; then + MACHER_PROG="$ac_cv_path_macher" + AC_MSG_RESULT([$MACHER_PROG]) + AC_MSG_RESULT([Found macher in environment]) + fi + AC_MSG_CHECKING([for zip]) + AC_CACHE_VAL(ac_cv_path_zip, [ + search_path=`echo ${PATH} | sed -e 's/:/ /g'` + for dir in $search_path ; do + for j in `ls -r $dir/zip 2> /dev/null` \ + `ls -r $dir/zip 2> /dev/null` ; do + if test x"$ac_cv_path_zip" = x ; then + if test -f "$j" ; then + ac_cv_path_zip=$j + break + fi + fi + done + done + ]) + if test -f "$ac_cv_path_zip" ; then + ZIP_PROG="$ac_cv_path_zip" + AC_MSG_RESULT([$ZIP_PROG]) + ZIP_PROG_OPTIONS="-rq" + ZIP_PROG_VFSSEARCH="*" + AC_MSG_RESULT([Found INFO Zip in environment]) + # Use standard arguments for zip + else + # It is not an error if an installed version of Zip can't be located. + # We can use the locally distributed minizip instead + ZIP_PROG="./minizip${EXEEXT_FOR_BUILD}" + ZIP_PROG_OPTIONS="-o -r" + ZIP_PROG_VFSSEARCH="*" + ZIP_INSTALL_OBJS="minizip${EXEEXT_FOR_BUILD}" + AC_MSG_RESULT([No zip found on PATH. Building minizip]) + fi + AC_SUBST(MACHER_PROG) + AC_SUBST(ZIP_PROG) + AC_SUBST(ZIP_PROG_OPTIONS) + AC_SUBST(ZIP_PROG_VFSSEARCH) + AC_SUBST(ZIP_INSTALL_OBJS) +]) + # Local Variables: # mode: autoconf -# End: +# End: \ No newline at end of file diff --git a/manifest b/manifest index 032b39b8a9..bee34a7f2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sseen\swith\sMSVC. -D 2022-07-06T23:50:01.037 +C Upgrade\sthe\sTEA\sbuild\ssystem\sin\sautoconf/tea/.\sTo\smatch\stclconfig\scommit\s20fe9e6f59\sand\sTcl\sSample\sExtension\sbe47fb0446. +D 2022-07-07T20:49:22.140 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -19,15 +19,15 @@ F autoconf/Makefile.msc 8401a514e4e70add3c6448348ae31322d5cb7db427b05a20828f943c F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 42cfd21d0b19dc7d5d85fb5c405c5f3c6a4c923021c39128f6ba685355d8fd56 F autoconf/configure.ac ec7fa914c5e74ff212fe879f9bb6918e1234497e05facfb641f30c4d5893b277 -F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd +F autoconf/tea/Makefile.in 106a96f2f745d41a0f6193f1de98d7355830b65d45032c18cd7c90295ec24196 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.ac ea61e07340d97e4a79a081f0b8977198a6073edd060738dbb3ae5cb8d5e96f1c +F autoconf/tea/configure.ac 5cf3c7c68e000015372db91eebf11a9f353174957cc192e2eac62d3fb1590624 F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 -F autoconf/tea/pkgIndex.tcl.in 3ef61715cf1c7bdcff56947ffadb26bc991ca39d +F autoconf/tea/pkgIndex.tcl.in b9eb6dd37f64e08e637d576b3c83259814b9cddd78bec4af2e5abfc6c5c750ce F autoconf/tea/tclconfig/install-sh bdd5e293591621ae60d9824d86a4b1c5f22c3d00 -F autoconf/tea/tclconfig/tcl.m4 66ddf0a5d5e4b1d29bff472c0985fd7fa89d0fb5 +F autoconf/tea/tclconfig/tcl.m4 debe13280bd5a9d76dc34e7919cd9ed3a1408c7320400900357128c2d1abb723 F autoconf/tea/win/makefile.vc a5ff708245260c2794c6aaa0151efe5403d5896566eaf096747be0d9075284e4 F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea00342d1a1a7eaa19cb F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 @@ -1979,8 +1979,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 587795d47fcaf5142526fabbcc4d5a632f561f258414c2846e8932a49b5b2e6b -R 2eb751598396b97b93bf551986a07087 -U mistachkin -Z 4d05d451769cfa8ea2dbf377223e0684 +P 61e2094afbbcbd5fdf5c3ec06b96134fafb7b854dc9bfa7d0619bed6d35efbe4 +R b26c8f36db62c20aad24f24827303d40 +T *branch * tea-upgrade +T *sym-tea-upgrade * +T -sym-trunk * +U dan +Z 6408e9ff7d53d4b2eaa313ad961076a7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a0ba389f3a..743b0e6230 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61e2094afbbcbd5fdf5c3ec06b96134fafb7b854dc9bfa7d0619bed6d35efbe4 \ No newline at end of file +1531f7391890d7d3cd091c2d1284230f128e5282bf676967ebcb212210e51e71 \ No newline at end of file From f15b77b7cfdad810bdac026d13ff6aef31022f21 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 7 Jul 2022 21:04:03 +0000 Subject: [PATCH 037/151] Size reduction and performance increase in defragementPage() of btree.c. FossilOrigin-Name: 1b03f197b5572084177012a58990f8dba7ff10382ff5657fda62867a4d0b1af9 --- manifest | 12 +++++------ manifest.uuid | 2 +- src/btree.c | 58 +++++++++++++++++++++++++-------------------------- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 06040bca47..833eb98008 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Size\sreduction\sand\sperformance\sincrease\sin\sthe\spageFindSlot()\sroutine\sof\nbtree.c. -D 2022-07-07T20:29:49.982 +C Size\sreduction\sand\sperformance\sincrease\sin\sdefragementPage()\sof\sbtree.c. +D 2022-07-07T21:04:03.223 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 6599d972b674cbbcb76e9a9db0e206d3bf9b60186464c08ef82ab6fda0622d44 +F src/btree.c 85e73fcdad6ca71caf083427d7a84ee31bce23a844f0ca3dd41f176f905090c9 F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c 23f874642825d7eaaeeb7a3281b2b1a75e1d4c4dd9ae4dceddcd908266634214 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7d7aed053f600659c63d8bac6d5da77879936c3fc57bfd058a4943b3bc530575 -R 573750a14fc9838f7267df2add226c79 +P 5d247e38560b97975568e8a48324acaca1002ec2ef7fa3efc9c8aa7d83412aad +R a83c6075a9fd94646d9131a381899d86 U drh -Z 6e9989a326ed7411def6db942d22ad53 +Z fd00ac09a5e7246ba9e920ca18028a40 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5f4865c9bd..a69143fe3c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d247e38560b97975568e8a48324acaca1002ec2ef7fa3efc9c8aa7d83412aad \ No newline at end of file +1b03f197b5572084177012a58990f8dba7ff10382ff5657fda62867a4d0b1af9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2d2a20cd52..bdcf5b1478 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1513,7 +1513,6 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = 0; src = data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; @@ -1568,39 +1567,38 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ cbrk = usableSize; iCellLast = usableSize - 4; iCellStart = get2byte(&data[hdr+5]); - for(i=0; iiCellLast ){ - return SQLITE_CORRUPT_PAGE(pPage); + if( nCell>0 ){ + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart); + src = temp; + for(i=0; iiCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pc>=iCellStart && pc<=iCellLast ); + size = pPage->xCellSize(pPage, &src[pc]); + cbrk -= size; + if( cbrkusableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( cbrk+size<=usableSize && cbrk>=iCellStart ); + testcase( cbrk+size==usableSize ); + testcase( pc+size==usableSize ); + put2byte(pAddr, cbrk); + memcpy(&data[cbrk], &src[pc], size); } - assert( pc>=iCellStart && pc<=iCellLast ); - size = pPage->xCellSize(pPage, &src[pc]); - cbrk -= size; - if( cbrkusableSize ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - assert( cbrk+size<=usableSize && cbrk>=iCellStart ); - testcase( cbrk+size==usableSize ); - testcase( pc+size==usableSize ); - put2byte(pAddr, cbrk); - if( temp==0 ){ - if( cbrk==pc ) continue; - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart); - src = temp; - } - memcpy(&data[cbrk], &src[pc], size); } data[hdr+7] = 0; - defragment_out: +defragment_out: assert( pPage->nFree>=0 ); if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ return SQLITE_CORRUPT_PAGE(pPage); From e6aec7231206ee031668270379e95c0e8e65397c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 7 Jul 2022 22:59:35 +0000 Subject: [PATCH 038/151] Very small size reduction and performance increase in btree.c. FossilOrigin-Name: 9a7c031a822246ee36bc440c7492e9c8ebeec6bc7066e56f114756b1881b2eb8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 7 +++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 833eb98008..d5c95a678f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Size\sreduction\sand\sperformance\sincrease\sin\sdefragementPage()\sof\sbtree.c. -D 2022-07-07T21:04:03.223 +C Very\ssmall\ssize\sreduction\sand\sperformance\sincrease\sin\sbtree.c. +D 2022-07-07T22:59:35.373 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 85e73fcdad6ca71caf083427d7a84ee31bce23a844f0ca3dd41f176f905090c9 +F src/btree.c 1307d57f65023c9f37c2b6c62253343fca63877f161f694a593b14c0883cedda F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c 23f874642825d7eaaeeb7a3281b2b1a75e1d4c4dd9ae4dceddcd908266634214 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d247e38560b97975568e8a48324acaca1002ec2ef7fa3efc9c8aa7d83412aad -R a83c6075a9fd94646d9131a381899d86 +P 1b03f197b5572084177012a58990f8dba7ff10382ff5657fda62867a4d0b1af9 +R 90dc3f67b9cbf22f3b538962187867f6 U drh -Z fd00ac09a5e7246ba9e920ca18028a40 +Z d64fafa6d8a0e54f3ee1db1b4990dc40 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a69143fe3c..a7662b414c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b03f197b5572084177012a58990f8dba7ff10382ff5657fda62867a4d0b1af9 \ No newline at end of file +9a7c031a822246ee36bc440c7492e9c8ebeec6bc7066e56f114756b1881b2eb8 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index bdcf5b1478..80bca66353 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5249,8 +5249,6 @@ const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ - BtShared *pBt = pCur->pBt; - assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageapPage[pCur->iPage] = pCur->pPage; pCur->ix = 0; pCur->iPage++; - return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags); + return getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur, + pCur->curPagerFlags); } #ifdef SQLITE_DEBUG @@ -5370,7 +5369,7 @@ static int moveToRoot(BtCursor *pCur){ } sqlite3BtreeClearCursor(pCur); } - rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->pPage, + rc = getAndInitPage(pCur->pBt, pCur->pgnoRoot, &pCur->pPage, 0, pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; From 5ca48c0b35d48d4a421f5c0475a2d8b75f820212 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 8 Jul 2022 11:14:19 +0000 Subject: [PATCH 039/151] Update the build options used for the TEA package to match those used to build the binaries at download.html. FossilOrigin-Name: 9d6d9dba6680b62c24f5671109f3c28b6829645bffc5ed4e92d016fd2ad02a3e --- autoconf/tea/configure.ac | 8 ++++++++ manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/autoconf/tea/configure.ac b/autoconf/tea/configure.ac index 90f418f3e4..3e17b8b0be 100644 --- a/autoconf/tea/configure.ac +++ b/autoconf/tea/configure.ac @@ -76,8 +76,16 @@ TEA_ADD_HEADERS([]) TEA_ADD_INCLUDES([]) TEA_ADD_LIBS([]) TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1]) +TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS4=1]) +TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS5=1]) TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1]) TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1]) +TEA_ADD_CFLAGS([-DSQLITE_ENABLE_GEOPOLY=1]) +TEA_ADD_CFLAGS([-DSQLITE_ENABLE_MATH_FUNCTIONS=1]) +TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DESERIALIZE=1]) +TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DBPAGE_VTAB=1]) +TEA_ADD_CFLAGS([-DSQLITE_ENABLE_BYTECODE_VTAB=1]) +TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DBSTAT_VTAB=1]) TEA_ADD_STUB_SOURCES([]) TEA_ADD_TCL_SOURCES([]) diff --git a/manifest b/manifest index bee34a7f2e..e7d1dc507b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Upgrade\sthe\sTEA\sbuild\ssystem\sin\sautoconf/tea/.\sTo\smatch\stclconfig\scommit\s20fe9e6f59\sand\sTcl\sSample\sExtension\sbe47fb0446. -D 2022-07-07T20:49:22.140 +C Update\sthe\sbuild\soptions\sused\sfor\sthe\sTEA\spackage\sto\smatch\sthose\sused\sto\sbuild\sthe\sbinaries\sat\sdownload.html. +D 2022-07-08T11:14:19.181 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -22,7 +22,7 @@ F autoconf/configure.ac ec7fa914c5e74ff212fe879f9bb6918e1234497e05facfb641f30c4d F autoconf/tea/Makefile.in 106a96f2f745d41a0f6193f1de98d7355830b65d45032c18cd7c90295ec24196 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.ac 5cf3c7c68e000015372db91eebf11a9f353174957cc192e2eac62d3fb1590624 +F autoconf/tea/configure.ac 2bbcc0f449361c5dfbcfc51ebadc5118e4c4e8c670674a10b6443356ecdb493e F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in b9eb6dd37f64e08e637d576b3c83259814b9cddd78bec4af2e5abfc6c5c750ce @@ -1979,11 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 61e2094afbbcbd5fdf5c3ec06b96134fafb7b854dc9bfa7d0619bed6d35efbe4 -R b26c8f36db62c20aad24f24827303d40 -T *branch * tea-upgrade -T *sym-tea-upgrade * -T -sym-trunk * +P 1531f7391890d7d3cd091c2d1284230f128e5282bf676967ebcb212210e51e71 +R 8233c02bb53fa3e4558e93749578c5f1 U dan -Z 6408e9ff7d53d4b2eaa313ad961076a7 +Z fa074dc669beccc97178d30527f50cd3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 743b0e6230..84db4d80e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1531f7391890d7d3cd091c2d1284230f128e5282bf676967ebcb212210e51e71 \ No newline at end of file +9d6d9dba6680b62c24f5671109f3c28b6829645bffc5ed4e92d016fd2ad02a3e \ No newline at end of file From 554cb87d96d94b9a901a8de16d90cfa697df4030 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 8 Jul 2022 11:21:30 +0000 Subject: [PATCH 040/151] Update options in the TEA package msvc makefile to match other builds. FossilOrigin-Name: d72e12b99dd1f8abdd855130adc27dbfc8794cfe9550a8495f68c4498a89d07e --- autoconf/tea/win/makefile.vc | 11 +++++++++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/autoconf/tea/win/makefile.vc b/autoconf/tea/win/makefile.vc index d92a8428bf..70f41f63b1 100644 --- a/autoconf/tea/win/makefile.vc +++ b/autoconf/tea/win/makefile.vc @@ -252,6 +252,17 @@ INCLUDES = $(SQL_INCLUDES) $(TCL_INCLUDES) -I"$(WINDIR)" \ BASE_CLFAGS = $(cflags) $(cdebug) $(crt) $(INCLUDES) \ -DSQLITE_3_SUFFIX_ONLY=1 -DSQLITE_ENABLE_RTREE=1 \ -DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1 + -DSQLITE_ENABLE_FTS4=1 \ + -DSQLITE_ENABLE_FTS5=1 \ + -DSQLITE_3_SUFFIX_ONLY=1 \ + -DSQLITE_ENABLE_RTREE=1 \ + -DSQLITE_ENABLE_GEOPOLY=1 \ + -DSQLITE_ENABLE_MATH_FUNCTIONS=1 \ + -DSQLITE_ENABLE_DESERIALIZE=1 \ + -DSQLITE_ENABLE_DBPAGE_VTAB=1 \ + -DSQLITE_ENABLE_BYTECODE_VTAB=1 \ + -DSQLITE_ENABLE_DBSTAT_VTAB=1 + CON_CFLAGS = $(cflags) $(cdebug) $(crt) -DCONSOLE -DSQLITE_ENABLE_FTS3=1 TCL_CFLAGS = -DBUILD_sqlite -DUSE_TCL_STUBS \ -DPACKAGE_VERSION="\"$(DOTVERSION)\"" $(BASE_CLFAGS) \ diff --git a/manifest b/manifest index e7d1dc507b..4b853dc063 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sbuild\soptions\sused\sfor\sthe\sTEA\spackage\sto\smatch\sthose\sused\sto\sbuild\sthe\sbinaries\sat\sdownload.html. -D 2022-07-08T11:14:19.181 +C Update\soptions\sin\sthe\sTEA\spackage\smsvc\smakefile\sto\smatch\sother\sbuilds. +D 2022-07-08T11:21:30.127 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -28,7 +28,7 @@ F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in b9eb6dd37f64e08e637d576b3c83259814b9cddd78bec4af2e5abfc6c5c750ce F autoconf/tea/tclconfig/install-sh bdd5e293591621ae60d9824d86a4b1c5f22c3d00 F autoconf/tea/tclconfig/tcl.m4 debe13280bd5a9d76dc34e7919cd9ed3a1408c7320400900357128c2d1abb723 -F autoconf/tea/win/makefile.vc a5ff708245260c2794c6aaa0151efe5403d5896566eaf096747be0d9075284e4 +F autoconf/tea/win/makefile.vc cb856bed4fefe9981c5e44e0a0272efba3e1030e0f9783da9387cc6ff71b688b F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea00342d1a1a7eaa19cb F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1531f7391890d7d3cd091c2d1284230f128e5282bf676967ebcb212210e51e71 -R 8233c02bb53fa3e4558e93749578c5f1 +P 9d6d9dba6680b62c24f5671109f3c28b6829645bffc5ed4e92d016fd2ad02a3e +R ce56e50ba35107c6def73a3ed3b309a4 U dan -Z fa074dc669beccc97178d30527f50cd3 +Z 763f99dda03ae6f94bf436543fae3155 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 84db4d80e9..e29fb0bff1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d6d9dba6680b62c24f5671109f3c28b6829645bffc5ed4e92d016fd2ad02a3e \ No newline at end of file +d72e12b99dd1f8abdd855130adc27dbfc8794cfe9550a8495f68c4498a89d07e \ No newline at end of file From 0e400f4e7d1087bc4f0fd14b86de85f898d6f5e4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 8 Jul 2022 11:55:54 +0000 Subject: [PATCH 041/151] Remove an obsolete statement from a comment. No changes to code. FossilOrigin-Name: 99afb3f41b923f06211d655d46c902da13c8f7aadf2421d6e93c048f5b49e660 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 3 --- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d5c95a678f..9d732df472 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Very\ssmall\ssize\sreduction\sand\sperformance\sincrease\sin\sbtree.c. -D 2022-07-07T22:59:35.373 +C Remove\san\sobsolete\sstatement\sfrom\sa\scomment.\s\sNo\schanges\sto\scode. +D 2022-07-08T11:55:54.035 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -570,7 +570,7 @@ F src/printf.c 6166a30417b05c5b2f82e1f183f75faa2926ad60531c0b688a57dbc951441a20 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c 84a8443e3723e908730d754f54df4e1dacc1eb7073c0bd79c9d5efe977a9f5b9 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 3c6558bb0678a4d1d086c30afbd933c1160356aff5285825f6cf0de66b81f16c +F src/select.c 061b628dd1d3025a4ef3278e7128ce148d5f6bf58b8c1173c3c9e118d16198c6 F src/shell.c.in 2b85128ca8ea13fc2dc32f971d628d9f688a324a30f469619b817ce490764fcb F src/sqlite.h.in 01573eae96721f2a8ee2a9e3b7140ceeba2e9c44350911890b89b8ff0dcf6781 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1b03f197b5572084177012a58990f8dba7ff10382ff5657fda62867a4d0b1af9 -R 90dc3f67b9cbf22f3b538962187867f6 +P 9a7c031a822246ee36bc440c7492e9c8ebeec6bc7066e56f114756b1881b2eb8 +R 3e7508022cb0498325fdd1393c541034 U drh -Z d64fafa6d8a0e54f3ee1db1b4990dc40 +Z 3aecf681069937d17dd627bbc558be00 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a7662b414c..26476fe834 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a7c031a822246ee36bc440c7492e9c8ebeec6bc7066e56f114756b1881b2eb8 \ No newline at end of file +99afb3f41b923f06211d655d46c902da13c8f7aadf2421d6e93c048f5b49e660 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 76b20d694a..133283b1f1 100644 --- a/src/select.c +++ b/src/select.c @@ -1820,9 +1820,6 @@ static void generateSortTail( ** Return a pointer to a string containing the 'declaration type' of the ** expression pExpr. The string may be treated as static by the caller. ** -** Also try to estimate the size of the returned value and return that -** result in *pEstWidth. -** ** The declaration type is the exact datatype definition extracted from the ** original CREATE TABLE statement if the expression is a column. The ** declaration type for a ROWID field is INTEGER. Exactly when an expression From 2337e97949f5f8b28dca625c0590c67430c8dc27 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 8 Jul 2022 13:57:45 +0000 Subject: [PATCH 042/151] Fix typo in autoconf/tea/win/makefile.vc. FossilOrigin-Name: 094b2aadd55b0bd93cab5e3172e3a3c1bea83cd312920ffc79088d4a30d2689a --- autoconf/tea/win/makefile.vc | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/autoconf/tea/win/makefile.vc b/autoconf/tea/win/makefile.vc index 70f41f63b1..da56e811fc 100644 --- a/autoconf/tea/win/makefile.vc +++ b/autoconf/tea/win/makefile.vc @@ -251,7 +251,7 @@ INCLUDES = $(SQL_INCLUDES) $(TCL_INCLUDES) -I"$(WINDIR)" \ -I"$(GENERICDIR)" -I"$(ROOT)\.." BASE_CLFAGS = $(cflags) $(cdebug) $(crt) $(INCLUDES) \ -DSQLITE_3_SUFFIX_ONLY=1 -DSQLITE_ENABLE_RTREE=1 \ - -DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1 + -DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1 \ -DSQLITE_ENABLE_FTS4=1 \ -DSQLITE_ENABLE_FTS5=1 \ -DSQLITE_3_SUFFIX_ONLY=1 \ diff --git a/manifest b/manifest index 4b853dc063..c2011d9bde 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\soptions\sin\sthe\sTEA\spackage\smsvc\smakefile\sto\smatch\sother\sbuilds. -D 2022-07-08T11:21:30.127 +C Fix\stypo\sin\sautoconf/tea/win/makefile.vc. +D 2022-07-08T13:57:45.139 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -28,7 +28,7 @@ F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in b9eb6dd37f64e08e637d576b3c83259814b9cddd78bec4af2e5abfc6c5c750ce F autoconf/tea/tclconfig/install-sh bdd5e293591621ae60d9824d86a4b1c5f22c3d00 F autoconf/tea/tclconfig/tcl.m4 debe13280bd5a9d76dc34e7919cd9ed3a1408c7320400900357128c2d1abb723 -F autoconf/tea/win/makefile.vc cb856bed4fefe9981c5e44e0a0272efba3e1030e0f9783da9387cc6ff71b688b +F autoconf/tea/win/makefile.vc 2c478a9a962e48b2bf9062734e04d7c63c556e217095419173f9d7938d7d78f7 F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea00342d1a1a7eaa19cb F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9d6d9dba6680b62c24f5671109f3c28b6829645bffc5ed4e92d016fd2ad02a3e -R ce56e50ba35107c6def73a3ed3b309a4 +P d72e12b99dd1f8abdd855130adc27dbfc8794cfe9550a8495f68c4498a89d07e +R bad6e21eb03a6dca5cb9c42e35791ce8 U dan -Z 763f99dda03ae6f94bf436543fae3155 +Z af7e3af8326bc996b6cb3fe7208f05f5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e29fb0bff1..3d2303f76a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d72e12b99dd1f8abdd855130adc27dbfc8794cfe9550a8495f68c4498a89d07e \ No newline at end of file +094b2aadd55b0bd93cab5e3172e3a3c1bea83cd312920ffc79088d4a30d2689a \ No newline at end of file From 72f38795b40f6d350296cb2cab2ebe929f50c0d9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 8 Jul 2022 16:56:47 +0000 Subject: [PATCH 043/151] In wherePathSolver(), defer initializing variables until they are actually needed, in case they are not needed. This gives a small performance increase. FossilOrigin-Name: 6f28a9652a3f1e5354afb76314db8f33f92f5e888e336d025f34881c0a8d64ae --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/where.c | 6 ++++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 1466559d45..5566b77b6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Upgrade\sthe\sTEA\sbuild\ssystem\sused\sto\sbuild\sthe\sTcl\spackage. -D 2022-07-08T15:06:07.834 +C In\swherePathSolver(),\sdefer\sinitializing\svariables\suntil\sthey\sare\sactually\nneeded,\sin\scase\sthey\sare\snot\sneeded.\s\sThis\sgives\sa\ssmall\sperformance\sincrease. +D 2022-07-08T16:56:47.610 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -657,7 +657,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 0f34033977b2275793c4330b2ebc3fa180a1baee06591cbc8f6e0d7aaa37988d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 9a44063e60d8f42dd9dc8147b8e8dcfc315bbd13e25c395211292c36d828c869 +F src/where.c 97a92f3ab73cf09cb9a4ff34f208bfdcd8dfd9bafb588e1eb15b47f9bdbde837 F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 F src/wherecode.c 0b09abfcb88c61c6a6984a3e065786631ff35495e9bdf865e6b74ab0a1299c5b F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 @@ -1979,9 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 99afb3f41b923f06211d655d46c902da13c8f7aadf2421d6e93c048f5b49e660 094b2aadd55b0bd93cab5e3172e3a3c1bea83cd312920ffc79088d4a30d2689a -R e23b050ce97a6cc9fcd8c96574e9ef0b -T +closed 094b2aadd55b0bd93cab5e3172e3a3c1bea83cd312920ffc79088d4a30d2689a -U dan -Z ebcd28726742e2f504d4057c1311108e +P ace65da8fa947926566037762ee9ff1b7686173eb7a50c7b55c4204171228b08 +R d2d96e6b5475a09297d787991573c2b0 +U drh +Z 21d18793c5a389d6c592bdf36ef9dca6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b157acfd2d..c5d3e916cf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ace65da8fa947926566037762ee9ff1b7686173eb7a50c7b55c4204171228b08 \ No newline at end of file +6f28a9652a3f1e5354afb76314db8f33f92f5e888e336d025f34881c0a8d64ae \ No newline at end of file diff --git a/src/where.c b/src/where.c index de6ea91e3d..2ad69fadf9 100644 --- a/src/where.c +++ b/src/where.c @@ -4769,9 +4769,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ LogEst rCost; /* Cost of path (pFrom+pWLoop) */ LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ - i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */ Bitmask maskNew; /* Mask of src visited by (..) */ - Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ + Bitmask revMask; /* Mask of rev-order loops for (..) */ if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; @@ -4790,7 +4790,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; + isOrdered = pFrom->isOrdered; if( isOrdered<0 ){ + revMask = 0; isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); From 4efa360f8989e08aead81737c8c0b64e8bf1a47d Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 8 Jul 2022 17:57:10 +0000 Subject: [PATCH 044/151] Size reduction and performance optimization in whereLoopAddBtreeIndex(). FossilOrigin-Name: 64d2312c121e3dabbc1f86effe9dfc57915395466ea2c3746821848f743b8d8f --- manifest | 12 +++++------ manifest.uuid | 2 +- src/where.c | 59 ++++++++++++++++++++++++++------------------------- 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/manifest b/manifest index 5566b77b6b..bfe584d9da 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\swherePathSolver(),\sdefer\sinitializing\svariables\suntil\sthey\sare\sactually\nneeded,\sin\scase\sthey\sare\snot\sneeded.\s\sThis\sgives\sa\ssmall\sperformance\sincrease. -D 2022-07-08T16:56:47.610 +C Size\sreduction\sand\sperformance\soptimization\sin\swhereLoopAddBtreeIndex(). +D 2022-07-08T17:57:10.112 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -657,7 +657,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 0f34033977b2275793c4330b2ebc3fa180a1baee06591cbc8f6e0d7aaa37988d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 97a92f3ab73cf09cb9a4ff34f208bfdcd8dfd9bafb588e1eb15b47f9bdbde837 +F src/where.c 6a308f341606d42bc2cefd94d9d6f709027e8584ef54e5e6ef0a1003269a1522 F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 F src/wherecode.c 0b09abfcb88c61c6a6984a3e065786631ff35495e9bdf865e6b74ab0a1299c5b F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ace65da8fa947926566037762ee9ff1b7686173eb7a50c7b55c4204171228b08 -R d2d96e6b5475a09297d787991573c2b0 +P 6f28a9652a3f1e5354afb76314db8f33f92f5e888e336d025f34881c0a8d64ae +R 6129c6298e998c308adf6a3cb0363d65 U drh -Z 21d18793c5a389d6c592bdf36ef9dca6 +Z 5685d150b29f18817c8e694ae059d017 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c5d3e916cf..b89caf9f10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f28a9652a3f1e5354afb76314db8f33f92f5e888e336d025f34881c0a8d64ae \ No newline at end of file +64d2312c121e3dabbc1f86effe9dfc57915395466ea2c3746821848f743b8d8f \ No newline at end of file diff --git a/src/where.c b/src/where.c index 2ad69fadf9..d28fbd2890 100644 --- a/src/where.c +++ b/src/where.c @@ -2971,38 +2971,39 @@ static int whereLoopAddBtreeIndex( if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS; }else if( eOp & WO_ISNULL ){ pNew->wsFlags |= WHERE_COLUMN_NULL; - }else if( eOp & (WO_GT|WO_GE) ){ - testcase( eOp & WO_GT ); - testcase( eOp & WO_GE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; - pNew->u.btree.nBtm = whereRangeVectorLen( - pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm - ); - pBtm = pTerm; - pTop = 0; - if( pTerm->wtFlags & TERM_LIKEOPT ){ - /* Range constraints that come from the LIKE optimization are - ** always used in pairs. */ - pTop = &pTerm[1]; - assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); - assert( pTop->wtFlags & TERM_LIKEOPT ); - assert( pTop->eOperator==WO_LT ); - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ - pNew->aLTerm[pNew->nLTerm++] = pTop; - pNew->wsFlags |= WHERE_TOP_LIMIT; - pNew->u.btree.nTop = 1; - } }else{ - assert( eOp & (WO_LT|WO_LE) ); - testcase( eOp & WO_LT ); - testcase( eOp & WO_LE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; - pNew->u.btree.nTop = whereRangeVectorLen( + int nVecLen = whereRangeVectorLen( pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm ); - pTop = pTerm; - pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? - pNew->aLTerm[pNew->nLTerm-2] : 0; + if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pNew->u.btree.nBtm = nVecLen; + pBtm = pTerm; + pTop = 0; + if( pTerm->wtFlags & TERM_LIKEOPT ){ + /* Range constraints that come from the LIKE optimization are + ** always used in pairs. */ + pTop = &pTerm[1]; + assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); + assert( pTop->wtFlags & TERM_LIKEOPT ); + assert( pTop->eOperator==WO_LT ); + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTop; + pNew->wsFlags |= WHERE_TOP_LIMIT; + pNew->u.btree.nTop = 1; + } + }else{ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pNew->u.btree.nTop = nVecLen; + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aLTerm[pNew->nLTerm-2] : 0; + } } /* At this point pNew->nOut is set to the number of rows expected to From 5f289e8400e87b018c5acce2b7c17fec010f0bdf Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 8 Jul 2022 18:23:04 +0000 Subject: [PATCH 045/151] Omit an unnecessary structure initialization in whereLoopAddAll(), replacing it with an assert() to show that the initialization has already occurred, for a small performance increase and size reduction. FossilOrigin-Name: 27be9e963459ca0b20c339264125e175550debf50fdbec7d78412768a6326b42 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index bfe584d9da..1708bee800 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Size\sreduction\sand\sperformance\soptimization\sin\swhereLoopAddBtreeIndex(). -D 2022-07-08T17:57:10.112 +C Omit\san\sunnecessary\sstructure\sinitialization\sin\swhereLoopAddAll(),\sreplacing\nit\swith\san\sassert()\sto\sshow\sthat\sthe\sinitialization\shas\salready\soccurred,\sfor\na\ssmall\sperformance\sincrease\sand\ssize\sreduction. +D 2022-07-08T18:23:04.754 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -657,7 +657,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 0f34033977b2275793c4330b2ebc3fa180a1baee06591cbc8f6e0d7aaa37988d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 6a308f341606d42bc2cefd94d9d6f709027e8584ef54e5e6ef0a1003269a1522 +F src/where.c 54e534ef371b26ac9e4955a35fbefb368fad6ed3cf10abf3a5d748bbaf7f992b F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 F src/wherecode.c 0b09abfcb88c61c6a6984a3e065786631ff35495e9bdf865e6b74ab0a1299c5b F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6f28a9652a3f1e5354afb76314db8f33f92f5e888e336d025f34881c0a8d64ae -R 6129c6298e998c308adf6a3cb0363d65 +P 64d2312c121e3dabbc1f86effe9dfc57915395466ea2c3746821848f743b8d8f +R 8acd37a15a19ed816e7bea6945b5086b U drh -Z 5685d150b29f18817c8e694ae059d017 +Z d803b380ec1c17178dfbac162011dfa0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b89caf9f10..2f09d2a86d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -64d2312c121e3dabbc1f86effe9dfc57915395466ea2c3746821848f743b8d8f \ No newline at end of file +27be9e963459ca0b20c339264125e175550debf50fdbec7d78412768a6326b42 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d28fbd2890..14a8278329 100644 --- a/src/where.c +++ b/src/where.c @@ -4174,7 +4174,13 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ /* Loop over the tables in the join, from left to right */ pNew = pBuilder->pNew; - whereLoopInit(pNew); + + /* Verify that pNew has already been initialized */ + assert( pNew->nLTerm==0 ); + assert( pNew->wsFlags==0 ); + assert( pNew->nLSlot==ArraySize(pNew->aLTermSpace) ); + assert( pNew->aLTerm==pNew->aLTermSpace ); + pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT; for(iTab=0, pItem=pTabList->a; pItem Date: Fri, 8 Jul 2022 20:03:41 +0000 Subject: [PATCH 046/151] Performance optimizations in the WHERE clause processing of the query planner. FossilOrigin-Name: 50c8e8de52a6ba328bc9eb38e1a67033b643ddf41ff507d2240e97e0072b6db3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 26 +++++++++++++++++++------- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 1708bee800..a74381f053 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\san\sunnecessary\sstructure\sinitialization\sin\swhereLoopAddAll(),\sreplacing\nit\swith\san\sassert()\sto\sshow\sthat\sthe\sinitialization\shas\salready\soccurred,\sfor\na\ssmall\sperformance\sincrease\sand\ssize\sreduction. -D 2022-07-08T18:23:04.754 +C Performance\soptimizations\sin\sthe\sWHERE\sclause\sprocessing\sof\sthe\squery\splanner. +D 2022-07-08T20:03:41.347 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -657,7 +657,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 0f34033977b2275793c4330b2ebc3fa180a1baee06591cbc8f6e0d7aaa37988d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 54e534ef371b26ac9e4955a35fbefb368fad6ed3cf10abf3a5d748bbaf7f992b +F src/where.c 5af9a9773a3bb84acf4eac8b92b7951b780ece1dca914223f78effc4effeca2d F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 F src/wherecode.c 0b09abfcb88c61c6a6984a3e065786631ff35495e9bdf865e6b74ab0a1299c5b F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 64d2312c121e3dabbc1f86effe9dfc57915395466ea2c3746821848f743b8d8f -R 8acd37a15a19ed816e7bea6945b5086b +P 27be9e963459ca0b20c339264125e175550debf50fdbec7d78412768a6326b42 +R b437fa10f8f11540296768d07008e6e8 U drh -Z d803b380ec1c17178dfbac162011dfa0 +Z 9a58f65092cc6ff74de4cfec7aa43ed3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2f09d2a86d..4227294a2f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27be9e963459ca0b20c339264125e175550debf50fdbec7d78412768a6326b42 \ No newline at end of file +50c8e8de52a6ba328bc9eb38e1a67033b643ddf41ff507d2240e97e0072b6db3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 14a8278329..405811a566 100644 --- a/src/where.c +++ b/src/where.c @@ -2196,12 +2196,18 @@ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ } /* -** Deallocate internal memory used by a WhereLoop object +** Deallocate internal memory used by a WhereLoop object. Leave the +** object in an initialized state, as if it had been newly allocated. */ static void whereLoopClear(sqlite3 *db, WhereLoop *p){ - if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm); + if( p->aLTerm!=p->aLTermSpace ){ + sqlite3DbFreeNN(db, p->aLTerm); + p->aLTerm = p->aLTermSpace; + p->nLSlot = ArraySize(p->aLTermSpace); + } whereLoopClearUnion(db, p); - whereLoopInit(p); + p->nLTerm = 0; + p->wsFlags = 0; } /* @@ -2225,7 +2231,9 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ */ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ whereLoopClearUnion(db, pTo); - if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ + if( pFrom->nLTerm > pTo->nLSlot + && whereLoopResize(db, pTo, pFrom->nLTerm) + ){ memset(pTo, 0, WHERE_LOOP_XFER_SZ); return SQLITE_NOMEM_BKPT; } @@ -2878,7 +2886,11 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nBtm = saved_nBtm; pNew->u.btree.nTop = saved_nTop; pNew->nLTerm = saved_nLTerm; - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + if( pNew->nLTerm>=pNew->nLSlot + && whereLoopResize(db, pNew, pNew->nLTerm+1) + ){ + break; /* OOM while trying to enlarge the pNew->aLTerm array */ + } pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; @@ -4178,8 +4190,8 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ /* Verify that pNew has already been initialized */ assert( pNew->nLTerm==0 ); assert( pNew->wsFlags==0 ); - assert( pNew->nLSlot==ArraySize(pNew->aLTermSpace) ); - assert( pNew->aLTerm==pNew->aLTermSpace ); + assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) ); + assert( pNew->aLTerm!=0 ); pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT; for(iTab=0, pItem=pTabList->a; pItem Date: Sun, 10 Jul 2022 21:12:54 +0000 Subject: [PATCH 047/151] When an OOM occurs and sets the Parse.nErr value, also set the Parse.nErr value for all outer Parse objects. dbsqlfuzz d33f60aaa67733aa700cd69dacf8e0e23a327a29 FossilOrigin-Name: 9a494d25944f4f640026e7a7ae2948e555d7af31487c55ed2ec2818a0789b887 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a74381f053..aca3915202 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimizations\sin\sthe\sWHERE\sclause\sprocessing\sof\sthe\squery\splanner. -D 2022-07-08T20:03:41.347 +C When\san\sOOM\soccurs\sand\ssets\sthe\sParse.nErr\svalue,\salso\sset\sthe\sParse.nErr\nvalue\sfor\sall\souter\sParse\sobjects.\ndbsqlfuzz\sd33f60aaa67733aa700cd69dacf8e0e23a327a29 +D 2022-07-10T21:12:54.800 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -535,7 +535,7 @@ F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b F src/main.c 62100512f2f86e8af2e17cc41644a358868d5b26c691118b6ef31fea066d49d3 -F src/malloc.c a9127efdcef92d6934c6339ea9813075b90edc0ce2e5c723556381a3828fb720 +F src/malloc.c 4a3785323104678a8b4b0a482fe0c2a80900e7468ddf76ab0f2ea1c79a8ca8cd F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27be9e963459ca0b20c339264125e175550debf50fdbec7d78412768a6326b42 -R b437fa10f8f11540296768d07008e6e8 +P 50c8e8de52a6ba328bc9eb38e1a67033b643ddf41ff507d2240e97e0072b6db3 +R 5579bda63c3bf716f97570c4cdf5c935 U drh -Z 9a58f65092cc6ff74de4cfec7aa43ed3 +Z deb778e25ff7e8e8427aa4cca5b3d5e3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4227294a2f..af88024994 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -50c8e8de52a6ba328bc9eb38e1a67033b643ddf41ff507d2240e97e0072b6db3 \ No newline at end of file +9a494d25944f4f640026e7a7ae2948e555d7af31487c55ed2ec2818a0789b887 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index cfda60a0b6..c508bf752c 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -776,8 +776,13 @@ void *sqlite3OomFault(sqlite3 *db){ } DisableLookaside; if( db->pParse ){ + Parse *pParse; sqlite3ErrorMsg(db->pParse, "out of memory"); db->pParse->rc = SQLITE_NOMEM_BKPT; + for(pParse=db->pParse->pOuterParse; pParse; pParse = pParse->pOuterParse){ + pParse->nErr++; + pParse->rc = SQLITE_NOMEM; + } } } return 0; From 3dfab26028cc5482bc4d29270cd47f41fd863743 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Jul 2022 14:26:38 +0000 Subject: [PATCH 048/151] Fix a problem in fts3 to do with deferred tokens and OR expressions. FossilOrigin-Name: d0bfe5c574dbc467ba97d96306d78a62a55b78dbbbe4d25aa962db6aa40406f6 --- ext/fts3/fts3.c | 7 ++++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 097338f547..7a3c735e80 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -4382,6 +4382,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ } pPhrase->doclist.pList = aOut; + assert( p1 && p2 ); if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ pPhrase->doclist.bFreeList = 1; pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); @@ -5568,9 +5569,9 @@ static int fts3EvalTestExpr( default: { #ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( pCsr->pDeferred - && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) - ){ + if( pCsr->pDeferred && (pExpr->bDeferred || ( + pExpr->iDocid==pCsr->iPrevId && pExpr->pPhrase->doclist.pList + ))){ Fts3Phrase *pPhrase = pExpr->pPhrase; assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); if( pExpr->bDeferred ){ diff --git a/manifest b/manifest index aca3915202..312d618cd9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\san\sOOM\soccurs\sand\ssets\sthe\sParse.nErr\svalue,\salso\sset\sthe\sParse.nErr\nvalue\sfor\sall\souter\sParse\sobjects.\ndbsqlfuzz\sd33f60aaa67733aa700cd69dacf8e0e23a327a29 -D 2022-07-10T21:12:54.800 +C Fix\sa\sproblem\sin\sfts3\sto\sdo\swith\sdeferred\stokens\sand\sOR\sexpressions. +D 2022-07-11T14:26:38.196 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -102,7 +102,7 @@ F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 6634a3854e70afa8710ee5e3a7253cd0f0c89d4cce207fcbfe2ead3bad1db7d5 +F ext/fts3/fts3.c 5577b9e88b492e6dab0592c2367d82ae3a634109a0903d0c9f9db69121da1d85 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h dafdc371f9fbab175744b06cfe019d5f040cdfdbd11fea752f5dc28d45b04c05 F ext/fts3/fts3_aux.c f0dc9bd98582615b7750218899bd0c729879b6bbf94d1be57ca1833ff49afc6f @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 50c8e8de52a6ba328bc9eb38e1a67033b643ddf41ff507d2240e97e0072b6db3 -R 5579bda63c3bf716f97570c4cdf5c935 -U drh -Z deb778e25ff7e8e8427aa4cca5b3d5e3 +P 9a494d25944f4f640026e7a7ae2948e555d7af31487c55ed2ec2818a0789b887 +R 08586b82e80f0d7399080c682457ecfb +U dan +Z c2ed34cd2c4344d8712dcb33cfd78057 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index af88024994..4f0d1b7e7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a494d25944f4f640026e7a7ae2948e555d7af31487c55ed2ec2818a0789b887 \ No newline at end of file +d0bfe5c574dbc467ba97d96306d78a62a55b78dbbbe4d25aa962db6aa40406f6 \ No newline at end of file From cd9e86307911a04a211451569cdcee06af4b26a1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Jul 2022 14:36:03 +0000 Subject: [PATCH 049/151] Ensure that the Parse.nErr flag is set following an SQLITE_TOOBIG error on a nested parse. Fix for the problem identified by [forum/forumpost/d5a82ba9eedee30c | forum post d5a82ba9eedee30c]. Also, remove unnecessary clearing of the Parse.zErrMsg field following a nested parse. FossilOrigin-Name: 44d77a7f807f5dc3e94e6cd88a27bea79257f0f2ccf332891bdaa4668d0bb987 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/build.c | 2 -- src/tokenize.c | 1 + 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 312d618cd9..7a7772b388 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sfts3\sto\sdo\swith\sdeferred\stokens\sand\sOR\sexpressions. -D 2022-07-11T14:26:38.196 +C Ensure\sthat\sthe\sParse.nErr\sflag\sis\sset\sfollowing\san\sSQLITE_TOOBIG\serror\son\na\snested\sparse.\s\sFix\sfor\sthe\sproblem\sidentified\sby\n[forum/forumpost/d5a82ba9eedee30c\s|\sforum\spost\sd5a82ba9eedee30c].\nAlso,\sremove\sunnecessary\sclearing\sof\sthe\sParse.zErrMsg\sfield\nfollowing\sa\snested\sparse. +D 2022-07-11T14:36:03.450 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -513,7 +513,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 1307d57f65023c9f37c2b6c62253343fca63877f161f694a593b14c0883cedda F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e -F src/build.c 23f874642825d7eaaeeb7a3281b2b1a75e1d4c4dd9ae4dceddcd908266634214 +F src/build.c 29fcc97af5197511788a571ed35a001eea472cbe3bcdbae88178e17fcafd4341 F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad @@ -634,7 +634,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c a38f52058b517929e264094abd0b5fd1e8e145a1aa43bc6f6a72ae5218f96c98 +F src/tokenize.c 36eb0799e487759bbe73e5742b82ee676f06cea2515ff578d03c59a74ccf2d6c F src/treeview.c 4d8eda242386ca9f47276c5b62cb41922587d4e27002acfb82c58e85dac4d93a F src/trigger.c 61bea163b1fa3039bc572ed8312461b978e5c527e5301f302b078f4c1ccdec6a F src/update.c c52a7991bece0453d22c77c08469512ee2f1391c12503fd347d1c939220c5877 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9a494d25944f4f640026e7a7ae2948e555d7af31487c55ed2ec2818a0789b887 -R 08586b82e80f0d7399080c682457ecfb -U dan -Z c2ed34cd2c4344d8712dcb33cfd78057 +P d0bfe5c574dbc467ba97d96306d78a62a55b78dbbbe4d25aa962db6aa40406f6 +R e2485993c6349272c79d485280ee9729 +U drh +Z d5e203512ebbc12e61da151fb9a99fc8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4f0d1b7e7b..3f2c1a3417 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0bfe5c574dbc467ba97d96306d78a62a55b78dbbbe4d25aa962db6aa40406f6 \ No newline at end of file +44d77a7f807f5dc3e94e6cd88a27bea79257f0f2ccf332891bdaa4668d0bb987 \ No newline at end of file diff --git a/src/build.c b/src/build.c index e243e795fe..31ab81b09b 100644 --- a/src/build.c +++ b/src/build.c @@ -331,8 +331,6 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); db->mDbFlags |= DBFLAG_PreferBuiltin; sqlite3RunParser(pParse, zSql); - sqlite3DbFree(db, pParse->zErrMsg); - pParse->zErrMsg = 0; db->mDbFlags = savedDbFlags; sqlite3DbFree(db, zSql); memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ); diff --git a/src/tokenize.c b/src/tokenize.c index f0c0cc1910..b147cdb40f 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -614,6 +614,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ mxSqlLen -= n; if( mxSqlLen<0 ){ pParse->rc = SQLITE_TOOBIG; + pParse->nErr++; break; } #ifndef SQLITE_OMIT_WINDOWFUNC From fc7f8f81daea563caa61d62c282812f4f6bbf9df Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Jul 2022 18:11:51 +0000 Subject: [PATCH 050/151] Back out the optimization at [1a8c2e54375ee2cf7] because there are some cases where it does not work. FossilOrigin-Name: fe39c8d5fd813308fb27a05ce257ff003d3c09c0372f500e8def5a528a2558b7 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/wal.c | 12 +++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 7a7772b388..1b23b1b58b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\sParse.nErr\sflag\sis\sset\sfollowing\san\sSQLITE_TOOBIG\serror\son\na\snested\sparse.\s\sFix\sfor\sthe\sproblem\sidentified\sby\n[forum/forumpost/d5a82ba9eedee30c\s|\sforum\spost\sd5a82ba9eedee30c].\nAlso,\sremove\sunnecessary\sclearing\sof\sthe\sParse.zErrMsg\sfield\nfollowing\sa\snested\sparse. -D 2022-07-11T14:36:03.450 +C Back\sout\sthe\soptimization\sat\s[1a8c2e54375ee2cf7]\sbecause\sthere\sare\ssome\ncases\swhere\sit\sdoes\snot\swork. +D 2022-07-11T18:11:51.750 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -654,7 +654,7 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8 F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c F src/vtab.c 3d72c780d1ea08906a198e4f033921a658a54590e3ed72c544995d84f3f9464a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 0f34033977b2275793c4330b2ebc3fa180a1baee06591cbc8f6e0d7aaa37988d +F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 5af9a9773a3bb84acf4eac8b92b7951b780ece1dca914223f78effc4effeca2d @@ -1979,8 +1979,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d0bfe5c574dbc467ba97d96306d78a62a55b78dbbbe4d25aa962db6aa40406f6 -R e2485993c6349272c79d485280ee9729 +P 44d77a7f807f5dc3e94e6cd88a27bea79257f0f2ccf332891bdaa4668d0bb987 +Q -1a8c2e54375ee2cf73773b798fed0ae07b42f5e068fddc513c093de5c1f46615 +R fa1e214aa3098781f8133105e874fb59 U drh -Z d5e203512ebbc12e61da151fb9a99fc8 +Z 81ca1352813c96809ca09ee1d58005a6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f2c1a3417..4c5e4745b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -44d77a7f807f5dc3e94e6cd88a27bea79257f0f2ccf332891bdaa4668d0bb987 \ No newline at end of file +fe39c8d5fd813308fb27a05ce257ff003d3c09c0372f500e8def5a528a2558b7 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index e7c1c053bd..fdc4ac39b6 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3138,17 +3138,20 @@ int sqlite3WalFindFrame( u32 *piRead /* OUT: Frame number (or zero) */ ){ u32 iRead = 0; /* If !=0, WAL frame to return data from */ - u32 iLast; /* Last page in WAL for this reader */ + u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ int iMinHash; /* This routine is only be called from within a read transaction. */ assert( pWal->readLock>=0 || pWal->lockError ); - /* if pWal->readLock==0, then the WAL is ignored by the reader - ** so return early, as if the WAL were empty. + /* If the "last page" field of the wal-index header snapshot is 0, then + ** no data will be read from the wal under any circumstances. Return early + ** in this case as an optimization. Likewise, if pWal->readLock==0, + ** then the WAL is ignored by the reader so return early, as if the + ** WAL were empty. */ - if( pWal->readLock==0 && pWal->bShmUnreliable==0 ){ + if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ *piRead = 0; return SQLITE_OK; } @@ -3179,7 +3182,6 @@ int sqlite3WalFindFrame( ** table after the current read-transaction had started. */ iMinHash = walFramePage(pWal->minFrame); - iLast = pWal->hdr.mxFrame; for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){ WalHashLoc sLoc; /* Hash table location */ int iKey; /* Hash slot index */ From f1ab642cde13954f4e739fbb8f2705350821015d Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Jul 2022 18:26:14 +0000 Subject: [PATCH 051/151] Fix harmless compiler warnings about unused debugging functions in treeview.c. FossilOrigin-Name: 4d6f907712e35eddf6af36eb823c3ccdfcdff1c63b2c224b3bcf34ffec95d511 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/main.c | 1 + src/sqliteInt.h | 2 ++ src/treeview.c | 6 ++++++ 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1b23b1b58b..43ac201263 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Back\sout\sthe\soptimization\sat\s[1a8c2e54375ee2cf7]\sbecause\sthere\sare\ssome\ncases\swhere\sit\sdoes\snot\swork. -D 2022-07-11T18:11:51.750 +C Fix\sharmless\scompiler\swarnings\sabout\sunused\sdebugging\sfunctions\sin\streeview.c. +D 2022-07-11T18:26:14.973 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -534,7 +534,7 @@ F src/insert.c 173845e5a6bac96ae937409e4f876b631f26b31dabb9df8fd0eb3b130b2bb3a7 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b -F src/main.c 62100512f2f86e8af2e17cc41644a358868d5b26c691118b6ef31fea066d49d3 +F src/main.c b91c7e71af6f33640c35b8239a285040aad8dfcfdaaf979152e743c0f8017ea8 F src/malloc.c 4a3785323104678a8b4b0a482fe0c2a80900e7468ddf76ab0f2ea1c79a8ca8cd F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -575,7 +575,7 @@ F src/shell.c.in 2b85128ca8ea13fc2dc32f971d628d9f688a324a30f469619b817ce490764fc F src/sqlite.h.in 01573eae96721f2a8ee2a9e3b7140ceeba2e9c44350911890b89b8ff0dcf6781 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d -F src/sqliteInt.h 8353e96646372efdb0795a13cd9949831b4992c928de8f5c43b2524e8a4c6e7b +F src/sqliteInt.h 059d5a017ebf488c7484f79ea507f56b2bf4bb700f340abf91c5d2227869f275 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4a3da6d77eeb3531cb0dbdf7047772a2a1b99f98c69e90ce009c75fe6328b2c0 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -635,7 +635,7 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 36eb0799e487759bbe73e5742b82ee676f06cea2515ff578d03c59a74ccf2d6c -F src/treeview.c 4d8eda242386ca9f47276c5b62cb41922587d4e27002acfb82c58e85dac4d93a +F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 F src/trigger.c 61bea163b1fa3039bc572ed8312461b978e5c527e5301f302b078f4c1ccdec6a F src/update.c c52a7991bece0453d22c77c08469512ee2f1391c12503fd347d1c939220c5877 F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 @@ -1979,9 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 44d77a7f807f5dc3e94e6cd88a27bea79257f0f2ccf332891bdaa4668d0bb987 -Q -1a8c2e54375ee2cf73773b798fed0ae07b42f5e068fddc513c093de5c1f46615 -R fa1e214aa3098781f8133105e874fb59 +P fe39c8d5fd813308fb27a05ce257ff003d3c09c0372f500e8def5a528a2558b7 +R 5bb2b0fcce4a89358c79b40b17c80161 U drh -Z 81ca1352813c96809ca09ee1d58005a6 +Z 724318672b57a46adc079d5ffb44d6e1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4c5e4745b0..85a69401bb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fe39c8d5fd813308fb27a05ce257ff003d3c09c0372f500e8def5a528a2558b7 \ No newline at end of file +4d6f907712e35eddf6af36eb823c3ccdfcdff1c63b2c224b3bcf34ffec95d511 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 6865121dd9..d905ac8ead 100644 --- a/src/main.c +++ b/src/main.c @@ -4116,6 +4116,7 @@ int sqlite3_test_control(int op, ...){ sqlite3ShowWindow(0); sqlite3ShowWinFunc(0); #endif + sqlite3ShowSelect(0); } #endif break; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 45e3453464..106a5d1475 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4474,6 +4474,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list); void sqlite3TreeViewSelect(TreeView*, const Select*, u8); void sqlite3TreeViewWith(TreeView*, const With*, u8); void sqlite3TreeViewUpsert(TreeView*, const Upsert*, u8); +#if TREETRACE_ENABLED void sqlite3TreeViewDelete(const With*, const SrcList*, const Expr*, const ExprList*,const Expr*, const Trigger*); void sqlite3TreeViewInsert(const With*, const SrcList*, @@ -4482,6 +4483,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list); void sqlite3TreeViewUpdate(const With*, const SrcList*, const ExprList*, const Expr*, int, const ExprList*, const Expr*, const Upsert*, const Trigger*); +#endif #ifndef SQLITE_OMIT_TRIGGER void sqlite3TreeViewTriggerStep(TreeView*, const TriggerStep*, u8, u8); void sqlite3TreeViewTrigger(TreeView*, const Trigger*, u8, u8); diff --git a/src/treeview.c b/src/treeview.c index 36bfc29d13..89a128dff6 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -999,6 +999,7 @@ void sqlite3TreeViewUpsert( sqlite3TreeViewPop(&pView); } +#if TREETRACE_ENABLED /* ** Generate a human-readable diagram of the data structure that go ** into generating an DELETE statement. @@ -1052,7 +1053,9 @@ void sqlite3TreeViewDelete( } sqlite3TreeViewPop(&pView); } +#endif /* TREETRACE_ENABLED */ +#if TREETRACE_ENABLED /* ** Generate a human-readable diagram of the data structure that go ** into generating an INSERT statement. @@ -1120,7 +1123,9 @@ void sqlite3TreeViewInsert( } sqlite3TreeViewPop(&pView); } +#endif /* TREETRACE_ENABLED */ +#if TREETRACE_ENABLED /* ** Generate a human-readable diagram of the data structure that go ** into generating an UPDATE statement. @@ -1196,6 +1201,7 @@ void sqlite3TreeViewUpdate( } sqlite3TreeViewPop(&pView); } +#endif /* TREETRACE_ENABLED */ #ifndef SQLITE_OMIT_TRIGGER /* From d5c2e084498eb5c235bf6fb8f85a9bc7d940035c Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Jul 2022 19:12:47 +0000 Subject: [PATCH 052/151] In the sqlite_stmt extension, store the result of strlen() in a 64-bit integer to avoid a compiler warning, even though we know that the length will always fit comfortably in 32 bits. FossilOrigin-Name: 3fe19452499afc8e6b38905e1ce7e9153adbfebf10dccf39da4b7f1b0cd24f05 --- ext/misc/stmt.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/misc/stmt.c b/ext/misc/stmt.c index ac0ae6f0f6..42f4d651d9 100644 --- a/ext/misc/stmt.c +++ b/ext/misc/stmt.c @@ -220,7 +220,7 @@ static int stmtFilter( ppRow = &pCur->pRow; for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){ const char *zSql = sqlite3_sql(p); - int nSql = zSql ? strlen(zSql)+1 : 0; + sqlite3_int64 nSql = zSql ? strlen(zSql)+1 : 0; StmtRow *pNew = (StmtRow*)sqlite3_malloc(sizeof(StmtRow) + nSql); if( pNew==0 ) return SQLITE_NOMEM; diff --git a/manifest b/manifest index 43ac201263..48d79d477a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sabout\sunused\sdebugging\sfunctions\sin\streeview.c. -D 2022-07-11T18:26:14.973 +C In\sthe\ssqlite_stmt\sextension,\sstore\sthe\sresult\sof\sstrlen()\sin\sa\s64-bit\ninteger\sto\savoid\sa\scompiler\swarning,\seven\sthough\swe\sknow\sthat\sthe\slength\nwill\salways\sfit\scomfortably\sin\s32\sbits. +D 2022-07-11T19:12:47.468 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -344,7 +344,7 @@ F ext/misc/shathree.c 7b17615869a495659f1569ada1d8d3d21b4a24614f2746d93cc87ef7c0 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 94df9bbfa514a563c1484f684a2df3d128a2f7209a84ca3ca100c68a0163e29f F ext/misc/sqlar.c 0ace5d3c10fe736dc584bf1159a36b8e2e60fab309d310cd8a0eecd9036621b6 -F ext/misc/stmt.c 72b23e1746bedcf6e36907e972383e7eb74940344da90b2965149739b55cc801 +F ext/misc/stmt.c 2d68161cb32de25b4510cb8662af2e1ffd4e41c6b7bfb3aa64a49468bb6d36c1 F ext/misc/templatevtab.c 8a16a91a5ceaccfcbd6aaaa56d46828806e460dd194965b3f77bf38f14b942c4 F ext/misc/totype.c fa4aedeb07f66169005dffa8de3b0a2b621779fd44f85c103228a42afa71853b F ext/misc/uint.c 053fed3bce2e89583afcd4bf804d75d659879bbcedac74d0fa9ed548839a030b @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe39c8d5fd813308fb27a05ce257ff003d3c09c0372f500e8def5a528a2558b7 -R 5bb2b0fcce4a89358c79b40b17c80161 +P 4d6f907712e35eddf6af36eb823c3ccdfcdff1c63b2c224b3bcf34ffec95d511 +R bbab4139cbd2870e339b88582449a0e8 U drh -Z 724318672b57a46adc079d5ffb44d6e1 +Z fc97bd4702bf992952d5c0a51a0eeac2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 85a69401bb..ac704ea9e0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d6f907712e35eddf6af36eb823c3ccdfcdff1c63b2c224b3bcf34ffec95d511 \ No newline at end of file +3fe19452499afc8e6b38905e1ce7e9153adbfebf10dccf39da4b7f1b0cd24f05 \ No newline at end of file From 90120b9f4a31b9f0d5be2c1b32920c6f0d61b7bd Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Jul 2022 19:47:57 +0000 Subject: [PATCH 053/151] Use 64-bit memory allocation APIs in the sqlite_stmt virtual table, to avoid harmless compiler warnings. FossilOrigin-Name: afb9e60ee3b194f33664a6722356e5d3c69ce1d4da1a0affd92b8e6aaf4dd2da --- ext/misc/stmt.c | 6 +++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/misc/stmt.c b/ext/misc/stmt.c index 42f4d651d9..1687f41978 100644 --- a/ext/misc/stmt.c +++ b/ext/misc/stmt.c @@ -101,7 +101,7 @@ static int stmtConnect( "CREATE TABLE x(sql,ncol,ro,busy,nscan,nsort,naidx,nstep," "reprep,run,mem)"); if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -123,7 +123,7 @@ static int stmtDisconnect(sqlite3_vtab *pVtab){ */ static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ stmt_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->db = ((stmt_vtab*)p)->db; @@ -221,7 +221,7 @@ static int stmtFilter( for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){ const char *zSql = sqlite3_sql(p); sqlite3_int64 nSql = zSql ? strlen(zSql)+1 : 0; - StmtRow *pNew = (StmtRow*)sqlite3_malloc(sizeof(StmtRow) + nSql); + StmtRow *pNew = (StmtRow*)sqlite3_malloc64(sizeof(StmtRow) + nSql); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(StmtRow)); diff --git a/manifest b/manifest index 48d79d477a..56d7dbb8ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\ssqlite_stmt\sextension,\sstore\sthe\sresult\sof\sstrlen()\sin\sa\s64-bit\ninteger\sto\savoid\sa\scompiler\swarning,\seven\sthough\swe\sknow\sthat\sthe\slength\nwill\salways\sfit\scomfortably\sin\s32\sbits. -D 2022-07-11T19:12:47.468 +C Use\s64-bit\smemory\sallocation\sAPIs\sin\sthe\ssqlite_stmt\svirtual\stable,\sto\savoid\nharmless\scompiler\swarnings. +D 2022-07-11T19:47:57.590 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -344,7 +344,7 @@ F ext/misc/shathree.c 7b17615869a495659f1569ada1d8d3d21b4a24614f2746d93cc87ef7c0 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 94df9bbfa514a563c1484f684a2df3d128a2f7209a84ca3ca100c68a0163e29f F ext/misc/sqlar.c 0ace5d3c10fe736dc584bf1159a36b8e2e60fab309d310cd8a0eecd9036621b6 -F ext/misc/stmt.c 2d68161cb32de25b4510cb8662af2e1ffd4e41c6b7bfb3aa64a49468bb6d36c1 +F ext/misc/stmt.c ed05ad78013edccd43f4fc33d8244001f6f886eb2dfd2106ba33616ac514c6fb F ext/misc/templatevtab.c 8a16a91a5ceaccfcbd6aaaa56d46828806e460dd194965b3f77bf38f14b942c4 F ext/misc/totype.c fa4aedeb07f66169005dffa8de3b0a2b621779fd44f85c103228a42afa71853b F ext/misc/uint.c 053fed3bce2e89583afcd4bf804d75d659879bbcedac74d0fa9ed548839a030b @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d6f907712e35eddf6af36eb823c3ccdfcdff1c63b2c224b3bcf34ffec95d511 -R bbab4139cbd2870e339b88582449a0e8 +P 3fe19452499afc8e6b38905e1ce7e9153adbfebf10dccf39da4b7f1b0cd24f05 +R 9c418d7d2871b88acc68dd2dd3026e6e U drh -Z fc97bd4702bf992952d5c0a51a0eeac2 +Z e0cb3e16109e36d9cf204ca651b9678d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ac704ea9e0..7cbc23b4f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3fe19452499afc8e6b38905e1ce7e9153adbfebf10dccf39da4b7f1b0cd24f05 \ No newline at end of file +afb9e60ee3b194f33664a6722356e5d3c69ce1d4da1a0affd92b8e6aaf4dd2da \ No newline at end of file From f703b42dc4912bd32bcaf1a36f454bbf1c953c8f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 11 Jul 2022 21:40:34 +0000 Subject: [PATCH 054/151] Update the expected error messages in some OOM test cases to account for [44d77a7f807]. FossilOrigin-Name: b3d6b3c3fc68dca7e20418eefa35ce3b583322b884b88a11c6773419f027a7a4 --- ext/fts5/test/fts5fault4.test | 8 ++++++-- ext/rtree/rtree3.test | 8 ++++++-- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- test/fts3fault.test | 8 ++++++-- test/malloc_common.tcl | 1 + 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test index 877e0228ad..1d0d5c9b7c 100644 --- a/ext/fts5/test/fts5fault4.test +++ b/ext/fts5/test/fts5fault4.test @@ -16,12 +16,16 @@ source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl set testprefix fts5fault4 -# If SQLITE_ENABLE_FTS3 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return } +set ::TMPDBERROR [list 1 \ + {unable to open a temporary database file for storing temporary tables} +] + #------------------------------------------------------------------------- # An OOM while dropping an fts5 table. # @@ -391,7 +395,7 @@ do_faultsim_test 14.1 -faults oom-t* -prep { } -body { db eval { ALTER TABLE "tbl one" RENAME TO "tbl two" } } -test { - faultsim_test_result {0 {}} + faultsim_test_result {0 {}} $::TMPDBERROR } finish_test diff --git a/ext/rtree/rtree3.test b/ext/rtree/rtree3.test index e37d18ee84..c980863dbb 100644 --- a/ext/rtree/rtree3.test +++ b/ext/rtree/rtree3.test @@ -23,6 +23,10 @@ ifcapable !rtree { return } +set ::TMPDBERROR [list 1 \ + {unable to open a temporary database file for storing temporary tables} +] + # Test summary: # # rtree3-1: Test OOM in simple CREATE TABLE, INSERT, DELETE and SELECT @@ -196,9 +200,9 @@ do_test rtree3-7.prep { do_faultsim_test rtree3-7 -faults oom-* -prep { faultsim_restore_and_reopen } -body { - execsql { ALTER TABLE rt RENAME TO rt2 } + execsql { ALTER TABLE rt RENAME TO rt2 } } -test { - faultsim_test_result {0 {}} + faultsim_test_result {0 {}} $::TMPDBERROR } do_faultsim_test rtree3-8 -faults oom-* -prep { diff --git a/manifest b/manifest index 56d7dbb8ff..3bb5c732d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\s64-bit\smemory\sallocation\sAPIs\sin\sthe\ssqlite_stmt\svirtual\stable,\sto\savoid\nharmless\scompiler\swarnings. -D 2022-07-11T19:47:57.590 +C Update\sthe\sexpected\serror\smessages\sin\ssome\sOOM\stest\scases\sto\saccount\sfor\s[44d77a7f807]. +D 2022-07-11T21:40:34.209 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -191,7 +191,7 @@ F ext/fts5/test/fts5eb.test a973baadac524dbbb4ad9b0e99030e12cabde2c6b28e0ac43729 F ext/fts5/test/fts5fault1.test d28a65caee75db6897c3cf1358c5230d3bb2a3bf7fb31062c19c7e5382b3d2bd F ext/fts5/test/fts5fault2.test 69c8fdbef830cd0d450908d4504d5bb86609e255af99c421c20a0756251fe344 F ext/fts5/test/fts5fault3.test da2f9e3e56ff5740d68ebdd6877c97089e7ed28ddff28a0da87a6afea27e5522 -F ext/fts5/test/fts5fault4.test 1c1db5fcfe59401e7833146100f1d8de284a0a686fac31ddac9fb56c459f725b +F ext/fts5/test/fts5fault4.test 87a10d0caee57da587c7588b0c8d25d2930197399b4812ad1e4d574c75324cee F ext/fts5/test/fts5fault5.test a336e4e11847de24c9497f80cce18e00bb3fab7fb11f97d04eb9af898900a762 F ext/fts5/test/fts5fault6.test a0fc0a8f99e4b16500c31dfc7e38e1defe0f1693ac47650517ac7b723b1956f8 F ext/fts5/test/fts5fault7.test 0acbec416edb24b8881f154e99c31e9ccf73f539cfcd164090be139e9e97ed4c @@ -418,7 +418,7 @@ F ext/rtree/rtree.c d7b4b8b81d8d54376a7f81de5be85ec58b37c11604bcf42984a8418b3415 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 F ext/rtree/rtree1.test d47f58832145fcfed9067bc457ca8664962196c4566c17a1ebd679367db55d11 F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d -F ext/rtree/rtree3.test 4ee5d7df86040efe3d8d84f141f2962a7745452200a7cba1db06f86d97050499 +F ext/rtree/rtree3.test 272594f88c344e973864008bbe4c71fd3a41a264c097d568593ee7886d83d409 F ext/rtree/rtree4.test 304de65d484540111b896827e4261815e5dca4ce28eeecd58be648cd73452c4b F ext/rtree/rtree5.test 49c9041d713d54560b315c2c7ef7207ee287eba1b20f8266968a06f2e55d3142 F ext/rtree/rtree6.test 9ce3691c1aac43070a9f194f0ebf54372db346c5a82241fd11b525ed53ce9f3f @@ -1025,7 +1025,7 @@ F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8 F test/fts3expr4.test f5b2832549f01b1f7f73389fa21d4b875499bc95bf7c8b36271844888c6a0938 F test/fts3expr5.test a5b9a053becbdb8e973fbf4d6d3abaabeb42d511d1848bd57931f3e0a1cf983e F test/fts3f.test 8c438d5e1cab526b0021988fb1dc70cf3597b006a33ffd6c955ee89929077fe3 -F test/fts3fault.test 798e45af84be7978ca33d5bdc94246eb44724db24174b5d8e9b1ac46c57fb08d +F test/fts3fault.test f4e1342acfe6d216a001490e8cd52afac1f9ffe4a11bbcdcb296129a45c5df45 F test/fts3fault2.test 6a17a11d8034b1c4eca9f3091649273d56c49ff049e2173df8060f94341e9da0 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3fuzz001.test e3c7b0ce9b04cc02281dcc96812a277f02df03cd7dc082055d87e11eb18aaf56 @@ -1234,7 +1234,7 @@ F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test 25897506da0098cea09b302ff432b0fb6d8002773c1e0fc9732aa8b444bfd455 F test/mallocL.test fb311ff80afddf3b1a75e52289081f4754d901dc F test/mallocM.test 78bbe9d3da84a5c679123cdb40d7b2010b18fc46e13897e4f253c6ba6fbff134 -F test/malloc_common.tcl aac62499b76be719fac31e7a3e54a7fd53272e7f +F test/malloc_common.tcl 806c50379cf4fa65008cd4d5af18273e5dac8ab62d1d4316c76aa2ecd2e54018 F test/malloctraceviewer.tcl b7a54595270c1d201abf1c3f3d461f27eaf24cdef623ad08a0fe5e411264c8a9 F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3fe19452499afc8e6b38905e1ce7e9153adbfebf10dccf39da4b7f1b0cd24f05 -R 9c418d7d2871b88acc68dd2dd3026e6e -U drh -Z e0cb3e16109e36d9cf204ca651b9678d +P afb9e60ee3b194f33664a6722356e5d3c69ce1d4da1a0affd92b8e6aaf4dd2da +R 8ea09e929549a140c3e6dd1310564461 +U dan +Z f4f04c769fa4ae18f60bb76e1371a738 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7cbc23b4f9..548db96f5d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afb9e60ee3b194f33664a6722356e5d3c69ce1d4da1a0affd92b8e6aaf4dd2da \ No newline at end of file +b3d6b3c3fc68dca7e20418eefa35ce3b583322b884b88a11c6773419f027a7a4 \ No newline at end of file diff --git a/test/fts3fault.test b/test/fts3fault.test index 707e884fa4..21defd282f 100644 --- a/test/fts3fault.test +++ b/test/fts3fault.test @@ -18,6 +18,10 @@ set ::testprefix fts3fault # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test ; return } +set ::TMPDBERROR [list 1 \ + {unable to open a temporary database file for storing temporary tables} +] + # Test error handling in the sqlite3Fts3Init() function. This is the # function that registers the FTS3 module and various support functions # with SQLite. @@ -49,7 +53,7 @@ do_faultsim_test 2 -prep { } -body { execsql { ALTER TABLE t1 RENAME TO t2 } } -test { - faultsim_test_result {0 {}} + faultsim_test_result {0 {}} $::TMPDBERROR } # Test error handling in the special case where a single prefix query @@ -194,7 +198,7 @@ do_faultsim_test 8.2 -faults oom-t* -prep { } -body { execsql { SELECT mit(matchinfo(t8, 's')) FROM t8 WHERE t8 MATCH 'a b c' } } -test { - faultsim_test_result {0 3} + faultsim_test_result {0 3} $::TMPDBERROR } do_faultsim_test 8.3 -prep { faultsim_restore_and_reopen diff --git a/test/malloc_common.tcl b/test/malloc_common.tcl index 2d0e57e4fc..7d3c942f77 100644 --- a/test/malloc_common.tcl +++ b/test/malloc_common.tcl @@ -664,6 +664,7 @@ proc do_write_test {name tbl sql} { if {$::DO_MALLOC_TEST } { set answers [list {1 {out of memory}} {0 {}}] + lappend answers [list 1 {unable to open a temporary database file for storing temporary tables}] if {$::DO_MALLOC_TEST==1} { set modes {100000 persistent} } else { From ebdf4c0240211dd21b27dc4d95c2f942b5dd1c72 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Jul 2022 22:20:39 +0000 Subject: [PATCH 055/151] Add the --config option to wapptest.tcl. The argument is a glob pattern. All configurations must match the glob pattern in order to run. The default value is "*". FossilOrigin-Name: 51255bad4c1fb6074f602586fabad675ae310b9c37f4b80ecf60e2959fb0db12 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/wapptest.tcl | 13 +++++++++++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3bb5c732d0..d5a469c1f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sexpected\serror\smessages\sin\ssome\sOOM\stest\scases\sto\saccount\sfor\s[44d77a7f807]. -D 2022-07-11T21:40:34.209 +C Add\sthe\s--config\soption\sto\swapptest.tcl.\s\sThe\sargument\sis\sa\sglob\spattern.\s\sAll\nconfigurations\smust\smatch\sthe\sglob\spattern\sin\sorder\sto\srun.\s\sThe\sdefault\svalue\nis\s"*". +D 2022-07-11T22:20:39.584 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1801,7 +1801,7 @@ F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 F test/walvfs.test bccb3e0d235ef85e276f491d34db32c9ada1ea67be8d9f10aabe7b30319ec656 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec -F test/wapptest.tcl 899594e25684861d5b0c0880fb012364def50ef8097041b8ddf74be5ba7fa270 x +F test/wapptest.tcl 8d69504451f613141fc0107a60c4fdcf59745cb3d421cdcd051deb6cdf36b288 x F test/where.test d13cd7c24e80009d2b54e2f7a8893c457afa49c64f99359c9eb3fe668ba1d9d4 F test/where2.test 03c21a11e7b90e2845fc3c8b4002fc44cc2797fa74c86ee47d70bd7ea4f29ed6 F test/where3.test 5b4ffc0ac2ea0fe92f02b1244b7531522fe4d7bccf6fa8741d54e82c10e67753 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P afb9e60ee3b194f33664a6722356e5d3c69ce1d4da1a0affd92b8e6aaf4dd2da -R 8ea09e929549a140c3e6dd1310564461 -U dan -Z f4f04c769fa4ae18f60bb76e1371a738 +P b3d6b3c3fc68dca7e20418eefa35ce3b583322b884b88a11c6773419f027a7a4 +R fba8b20c9fe2d30ccb99ea466f2ceaef +U drh +Z 2c62dc1d407b4391742c0258c96664b6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 548db96f5d..7b69f77625 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b3d6b3c3fc68dca7e20418eefa35ce3b583322b884b88a11c6773419f027a7a4 \ No newline at end of file +51255bad4c1fb6074f602586fabad675ae310b9c37f4b80ecf60e2959fb0db12 \ No newline at end of file diff --git a/test/wapptest.tcl b/test/wapptest.tcl index b7e16e7227..47eeee5f1d 100755 --- a/test/wapptest.tcl +++ b/test/wapptest.tcl @@ -8,6 +8,7 @@ source [file join [file dirname [info script]] wapp.tcl] # Variables set by the "control" form: # # G(platform) - User selected platform. +# G(cfgglob) - Glob pattern that all configurations must match # G(test) - Set to "Normal", "Veryquick", "Smoketest" or "Build-Only". # G(keep) - Boolean. True to delete no files after each test. # G(msvc) - Boolean. True to use MSVC as the compiler. @@ -15,6 +16,7 @@ source [file join [file dirname [info script]] wapp.tcl] # G(jobs) - How many sub-processes to run simultaneously. # set G(platform) $::tcl_platform(os)-$::tcl_platform(machine) +set G(cfgglob * set G(test) Normal set G(keep) 1 set G(msvc) 0 @@ -117,6 +119,10 @@ proc set_test_array {} { if {$G(debug)==0} { set debug "-nodebug"} foreach {config target} [releasetest_data tests $debug $G(platform)] { + # All configuration names must match $g(cfgglob), which defaults to * + # + if {![string match -nocase $G(cfgglob) $config]} continue + # If using MSVC, do not run sanitize or valgrind tests. Or the # checksymbols test. if {$G(msvc) && ( @@ -785,6 +791,7 @@ default it uses "wapp" to provide an interactive interface. Supported command line options (all optional) are: --platform PLATFORM (which tests to run) + --config GLOB (only run configurations matching GLOB) --smoketest (run "make smoketest" only) --veryquick (run veryquick.test only) --buildonly (build executables, do not run tests) @@ -876,6 +883,12 @@ for {set i 0} {$i < [llength $lTestArg]} {incr i} { set G(stdout) 1 } + -config { + if {$i==[llength $lTestArg]-1} { wapptest_usage } + incr i + set G(cfgglob) [lindex $lTestArg $i] + } + -stdout { set G(stdout) 1 } From b2d71371547de4b308292d94e8961264a74641c4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 12 Jul 2022 07:13:38 +0000 Subject: [PATCH 056/151] Back out the pager performance enhancement at [a1c090e08139f99d3], because it turns out we should never allow a zero key into the pcache interface according to the design specs, even if that page is immediately released without ever being used. FossilOrigin-Name: ec96293ead83603ebe5d7f250d6fdc11f22172f05a9513f175331437c3eaa4c8 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/pager.c | 3 ++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d5a469c1f3..e239de3960 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--config\soption\sto\swapptest.tcl.\s\sThe\sargument\sis\sa\sglob\spattern.\s\sAll\nconfigurations\smust\smatch\sthe\sglob\spattern\sin\sorder\sto\srun.\s\sThe\sdefault\svalue\nis\s"*". -D 2022-07-11T22:20:39.584 +C Back\sout\sthe\spager\sperformance\senhancement\sat\s[a1c090e08139f99d3],\sbecause\sit\nturns\sout\swe\sshould\snever\sallow\sa\szero\skey\sinto\sthe\spcache\sinterface\saccording\nto\sthe\sdesign\sspecs,\seven\sif\sthat\spage\sis\simmediately\sreleased\swithout\sever\nbeing\sused. +D 2022-07-12T07:13:38.289 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -557,7 +557,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 2df2b33db88f00af13805d4573ee126bc5973f9e3b91d03c575fa7ba64e7dc41 F src/os_win.c a8ea80037e81127ca01959daa87387cc135f325c88dc745376c4f760de852a10 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c a7cad005d788957737cb6662960f022d767dded35bcecf06dae090213d62e924 +F src/pager.c 74596fc3d5d8a50de32c37225fc300cccd5ea27ea303c5f4b845d6572f999c5f F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564 F src/pcache.c 084e638432c610f95aea72b8509f0845d2791293f39d1b82f0c0a7e089c3bb6b @@ -1979,8 +1979,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b3d6b3c3fc68dca7e20418eefa35ce3b583322b884b88a11c6773419f027a7a4 -R fba8b20c9fe2d30ccb99ea466f2ceaef +P 51255bad4c1fb6074f602586fabad675ae310b9c37f4b80ecf60e2959fb0db12 +Q -a1c090e08139f99d30aa89db0756dc59fe8990ce15b3db4d4b726cc6acdab46f +R a92351bd0b7826d19b4b1cf7b0fdbaec U drh -Z 2c62dc1d407b4391742c0258c96664b6 +Z 1a1f72772307e1446eb3c352af79bbe8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7b69f77625..8e767f9127 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -51255bad4c1fb6074f602586fabad675ae310b9c37f4b80ecf60e2959fb0db12 \ No newline at end of file +ec96293ead83603ebe5d7f250d6fdc11f22172f05a9513f175331437c3eaa4c8 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 82e9dc7a69..95e6eb8af1 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5495,6 +5495,7 @@ static int getPageNormal( assert( assert_pager_state(pPager) ); assert( pPager->hasHeldSharedLock==1 ); + if( pgno==0 ) return SQLITE_CORRUPT_BKPT; pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); if( pBase==0 ){ pPg = 0; @@ -5525,7 +5526,7 @@ static int getPageNormal( ** (*) obsolete. Was: maximum page number is 2^31 ** (2) Never try to fetch the locking page */ - if( pgno==0 || pgno==PAGER_SJ_PGNO(pPager) ){ + if( pgno==PAGER_SJ_PGNO(pPager) ){ rc = SQLITE_CORRUPT_BKPT; goto pager_acquire_err; } From ea1e3b425136d935c06de7883056e4b8a876be66 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 12 Jul 2022 09:40:27 +0000 Subject: [PATCH 057/151] Renamed SQLITE_SHELL_WASM_MODE to SQLITE_SHELL_WASM_WEB_MODE and no longer automatically enable it if __EMSCRIPTEN__ is defined, in order to facilitate using Emscripten to build the shell for CLI-based WASM runtimes (which cannot make use of the web-specific user input changes). The fiddle build now explicitly passes the new flag on at compile-time. FossilOrigin-Name: ee059ad5a811a1511e37158f041a7bf9070529d530410d2f1c4395cdd25c6d33 --- Makefile.in | 1 + manifest | 17 +++--- manifest.uuid | 2 +- src/shell.c.in | 149 ++++++++++++++++++++++++------------------------- 4 files changed, 84 insertions(+), 85 deletions(-) diff --git a/Makefile.in b/Makefile.in index e8f104c2a6..94fa96ea83 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1540,6 +1540,7 @@ $(fiddle_module_js): Makefile sqlite3.c shell.c \ emcc -o $@ $(emcc_flags) \ -sEXPORT_NAME=initFiddleModule \ -sEXPORTED_FUNCTIONS=@$(fiddle_dir_abs)/EXPORTED_FUNCTIONS.fiddle \ + -DSQLITE_SHELL_WASM_WEB_MODE \ sqlite3.c shell.c gzip < $@ > $@.gz gzip < $(fiddle_dir)/fiddle-module.wasm > $(fiddle_dir)/fiddle-module.wasm.gz diff --git a/manifest b/manifest index e239de3960..de0503af9a 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Back\sout\sthe\spager\sperformance\senhancement\sat\s[a1c090e08139f99d3],\sbecause\sit\nturns\sout\swe\sshould\snever\sallow\sa\szero\skey\sinto\sthe\spcache\sinterface\saccording\nto\sthe\sdesign\sspecs,\seven\sif\sthat\spage\sis\simmediately\sreleased\swithout\sever\nbeing\sused. -D 2022-07-12T07:13:38.289 +C Renamed\sSQLITE_SHELL_WASM_MODE\sto\sSQLITE_SHELL_WASM_WEB_MODE\sand\sno\slonger\sautomatically\senable\sit\sif\s__EMSCRIPTEN__\sis\sdefined,\sin\sorder\sto\sfacilitate\susing\sEmscripten\sto\sbuild\sthe\sshell\sfor\sCLI-based\sWASM\sruntimes\s(which\scannot\smake\suse\sof\sthe\sweb-specific\suser\sinput\schanges).\sThe\sfiddle\sbuild\snow\sexplicitly\spasses\sthe\snew\sflag\son\sat\scompile-time. +D 2022-07-12T09:40:27.255 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 731cf154c61e7cd63a2707600fecde816cedc113bd9901723522aefc74ddb292 +F Makefile.in ab82a7cf01fd4d2b1164c8cb20513b59f4223a1a6e713938b1fb5b842418020d F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc de7cb3e095ce2fdc33513ccd76ebdaeda1483d0ddab0410fe65cbdeadd4c0ee1 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -571,7 +571,7 @@ F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c 84a8443e3723e908730d754f54df4e1dacc1eb7073c0bd79c9d5efe977a9f5b9 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 061b628dd1d3025a4ef3278e7128ce148d5f6bf58b8c1173c3c9e118d16198c6 -F src/shell.c.in 2b85128ca8ea13fc2dc32f971d628d9f688a324a30f469619b817ce490764fcb +F src/shell.c.in b928ddd7c3b5c63a9b625695e8af6a8831efd267d5187b9418f9eca05157bda3 F src/sqlite.h.in 01573eae96721f2a8ee2a9e3b7140ceeba2e9c44350911890b89b8ff0dcf6781 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d @@ -1979,9 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 51255bad4c1fb6074f602586fabad675ae310b9c37f4b80ecf60e2959fb0db12 -Q -a1c090e08139f99d30aa89db0756dc59fe8990ce15b3db4d4b726cc6acdab46f -R a92351bd0b7826d19b4b1cf7b0fdbaec -U drh -Z 1a1f72772307e1446eb3c352af79bbe8 +P ec96293ead83603ebe5d7f250d6fdc11f22172f05a9513f175331437c3eaa4c8 +R 8020e393ac6ab4de8f603cb939b22712 +U stephan +Z d97d3fda1a826afffbfc7a98df7e436e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8e767f9127..924255a5e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec96293ead83603ebe5d7f250d6fdc11f22172f05a9513f175331437c3eaa4c8 \ No newline at end of file +ee059ad5a811a1511e37158f041a7bf9070529d530410d2f1c4395cdd25c6d33 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 4e921cb6e4..bc5e889a59 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -37,6 +37,16 @@ # define SQLITE_OS_WINRT 0 #endif +/* +** Use -DSQLITE_SHELL_WASM_WEB_MODE to build for Emscripten targeting +** web browser use. We do not automatically do that using +** ifdef(__EMSCRIPTEN__) because it should be possible to build the +** shell with Emscripten for use in shell-based WASM/WASI runtime +** environments. The browser-mode build has much different user input +** requirements and the "wasm web" build rewires the user input +** subsystem to account for that. +*/ + /* ** Warning pragmas copied from msvc.h in the core. */ @@ -229,17 +239,6 @@ static void setTextMode(FILE *file, int isOutput){ # define setTextMode(X,Y) #endif -/* -** When compiling with emcc (a.k.a. emscripten), we're building a -** WebAssembly (WASM) bundle and need to disable and rewire a few -** things. -*/ -#ifdef __EMSCRIPTEN__ -#define SQLITE_SHELL_WASM_MODE -#else -#undef SQLITE_SHELL_WASM_MODE -#endif - /* True if the timer is enabled */ static int enableTimer = 0; @@ -701,7 +700,7 @@ static char *local_getline(char *zLine, FILE *in){ ** be freed by the caller or else passed back into this routine via the ** zPrior argument for reuse. */ -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ char *zPrompt; char *zResult; @@ -721,7 +720,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ } return zResult; } -#endif /* !SQLITE_SHELL_WASM_MODE */ +#endif /* !SQLITE_SHELL_WASM_WEB_MODE */ /* ** Return the value of a hexadecimal digit. Return -1 if the input @@ -1027,7 +1026,7 @@ INCLUDE ../ext/misc/decimal.c INCLUDE ../ext/misc/ieee754.c INCLUDE ../ext/misc/series.c INCLUDE ../ext/misc/regexp.c -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE INCLUDE ../ext/misc/fileio.c INCLUDE ../ext/misc/completion.c INCLUDE ../ext/misc/appendvfs.c @@ -1162,7 +1161,7 @@ struct ShellState { char *zNonce; /* Nonce for temporary safe-mode excapes */ EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_WASM_WEB_MODE struct { const char * zInput; /* Input string from wasm/JS proxy */ const char * zPos; /* Cursor pos into zInput */ @@ -1170,7 +1169,7 @@ struct ShellState { #endif }; -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_WASM_WEB_MODE static ShellState shellState; #endif @@ -1838,7 +1837,7 @@ static int safeModeAuth( UNUSED_PARAMETER(zA4); switch( op ){ case SQLITE_ATTACH: { -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE /* In WASM builds the filesystem is a virtual sandbox, so ** there's no harm in using ATTACH. */ failIfSafeMode(p, "cannot run ATTACH in safe mode"); @@ -4252,7 +4251,7 @@ static int run_schema_dump_query( */ static const char *(azHelp[]) = { #if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) \ - && !defined(SQLITE_SHELL_WASM_MODE) + && !defined(SQLITE_SHELL_WASM_WEB_MODE) ".archive ... Manage SQL archives", " Each command must have exactly one of the following options:", " -c, --create Create a new archive", @@ -4278,7 +4277,7 @@ static const char *(azHelp[]) = { #ifndef SQLITE_OMIT_AUTHORIZATION ".auth ON|OFF Show authorizer callbacks", #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".backup ?DB? FILE Backup DB (default \"main\") to FILE", " Options:", " --append Use the appendvfs", @@ -4286,11 +4285,11 @@ static const char *(azHelp[]) = { #endif ".bail on|off Stop after hitting an error. Default OFF", ".binary on|off Turn binary output on or off. Default OFF", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".cd DIRECTORY Change the working directory to DIRECTORY", #endif ".changes on|off Show number of rows changed by SQL", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".check GLOB Fail if output since .testcase does not match", ".clone NEWDB Clone data into NEWDB from the existing database", #endif @@ -4316,11 +4315,11 @@ static const char *(azHelp[]) = { " trace Like \"full\" but enable \"PRAGMA vdbe_trace\"", #endif " trigger Like \"full\" but also show trigger bytecode", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".excel Display the output of next command in spreadsheet", " --bom Put a UTF8 byte-order mark on intermediate file", #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".exit ?CODE? Exit this program with return-code CODE", #endif ".expert EXPERIMENTAL. Suggest indexes for queries", @@ -4331,7 +4330,7 @@ static const char *(azHelp[]) = { ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", ".headers on|off Turn display of headers on or off", ".help ?-all? ?PATTERN? Show help text for PATTERN", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".import FILE TABLE Import data from FILE into TABLE", " Options:", " --ascii Use \\037 and \\036 as column and row separators", @@ -4360,10 +4359,10 @@ static const char *(azHelp[]) = { ".lint OPTIONS Report potential schema issues.", " Options:", " fkey-indexes Find missing foreign key indexes", -#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_WASM_MODE) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_WASM_WEB_MODE) ".load FILE ?ENTRY? Load an extension library", #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".log FILE|off Turn logging on or off. FILE can be stderr/stdout", #endif ".mode MODE ?OPTIONS? Set output mode", @@ -4390,11 +4389,11 @@ static const char *(azHelp[]) = { " --quote Quote output text as SQL literals", " --noquote Do not quote output text", " TABLE The name of SQL table used for \"insert\" mode", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".nonce STRING Suspend safe mode for one command if nonce matches", #endif ".nullvalue STRING Use STRING in place of NULL values", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", " If FILE begins with '|' then open as a pipe", " --bom Put a UTF8 byte-order mark at the beginning", @@ -4416,7 +4415,7 @@ static const char *(azHelp[]) = { " --nofollow Do not follow symbolic links", " --readonly Open FILE readonly", " --zip FILE is a ZIP archive", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".output ?FILE? Send output to FILE or stdout if FILE is omitted", " If FILE begins with '|' then open it as a pipe.", " Options:", @@ -4440,7 +4439,7 @@ static const char *(azHelp[]) = { " --reset Reset the count for each input and interrupt", #endif ".prompt MAIN CONTINUE Replace the standard prompts", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".quit Exit this program", ".read FILE Read input from FILE or command output", " If FILE begins with \"|\", it is a command that generates the input.", @@ -4453,7 +4452,7 @@ static const char *(azHelp[]) = { " --no-rowids Do not attempt to recover rowid values", " that are not also INTEGER PRIMARY KEYs", #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", ".save ?OPTIONS? FILE Write database to FILE (an alias for .backup ...)", #endif @@ -4490,7 +4489,7 @@ static const char *(azHelp[]) = { " --sha3-384 Use the sha3-384 algorithm", " --sha3-512 Use the sha3-512 algorithm", " Any other argument is a LIKE pattern for tables to hash", -#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE) +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_WEB_MODE) ".shell CMD ARGS... Run CMD ARGS... in a system shell", #endif ".show Show the current values for various settings", @@ -4499,11 +4498,11 @@ static const char *(azHelp[]) = { " on Turn on automatic stat display", " stmt Show statement stats", " vmstep Show the virtual machine step count only", -#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE) +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_WEB_MODE) ".system CMD ARGS... Run CMD ARGS... in a system shell", #endif ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE ".testcase NAME Begin redirecting output to 'testcase-out.txt'", #endif ".testctrl CMD ... Run various sqlite3_test_control() operations", @@ -5056,7 +5055,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_regexp_init(p->db, 0, 0); sqlite3_ieee_init(p->db, 0, 0); sqlite3_series_init(p->db, 0, 0); -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE sqlite3_fileio_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); #endif @@ -8186,7 +8185,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) \ - && !defined(SQLITE_SHELL_WASM_MODE) + && !defined(SQLITE_SHELL_WASM_WEB_MODE) if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){ open_db(p, 0); failIfSafeMode(p, "cannot run .archive in safe mode"); @@ -8194,7 +8193,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0) || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0) ){ @@ -8263,7 +8262,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } close_db(pDest); }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 ){ if( nArg==2 ){ @@ -8294,7 +8293,7 @@ static int do_meta_command(char *zLine, ShellState *p){ test_breakpoint(); }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( c=='c' && strcmp(azArg[0],"cd")==0 ){ failIfSafeMode(p, "cannot run .cd in safe mode"); if( nArg==2 ){ @@ -8314,7 +8313,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){ if( nArg==2 ){ @@ -8325,7 +8324,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE /* Cancel output redirection, if it is currently set (by .testcase) ** Then read the content of the testcase-out.txt file and compare against ** azArg[1]. If there are differences, report an error and exit. @@ -8350,9 +8349,9 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zRes); }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){ failIfSafeMode(p, "cannot run .clone in safe mode"); if( nArg==2 ){ @@ -8362,7 +8361,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ if( c=='c' && strncmp(azArg[0], "connection", n)==0 ){ if( nArg==1 ){ @@ -8651,7 +8650,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); rc = 2; @@ -8911,7 +8910,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( c=='i' && strncmp(azArg[0], "import", n)==0 ){ char *zTable = 0; /* Insert data into this table */ char *zSchema = 0; /* within this schema (may default to "main") */ @@ -9202,7 +9201,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sCtx.nRow, sCtx.nErr, sCtx.nLine-1); } }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ #ifndef SQLITE_UNTESTABLE if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){ @@ -9392,7 +9391,7 @@ static int do_meta_command(char *zLine, ShellState *p){ lintDotCommand(p, azArg, nArg); }else -#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_WASM_MODE) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_WASM_WEB_MODE) if( c=='l' && strncmp(azArg[0], "load", n)==0 ){ const char *zFile, *zProc; char *zErrMsg = 0; @@ -9414,7 +9413,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( c=='l' && strncmp(azArg[0], "log", n)==0 ){ failIfSafeMode(p, "cannot run .log in safe mode"); if( nArg!=2 ){ @@ -9551,7 +9550,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->cMode = p->mode; }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( c=='n' && strcmp(azArg[0], "nonce")==0 ){ if( nArg!=2 ){ raw_printf(stderr, "Usage: .nonce NONCE\n"); @@ -9566,7 +9565,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** at the end of this procedure */ } }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 ){ if( nArg==2 ){ @@ -9588,7 +9587,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* Check for command-line arguments */ for(iName=1; iNameszMax = integerValue(azArg[++iName]); #endif /* SQLITE_OMIT_DESERIALIZE */ }else -#endif /* !SQLITE_SHELL_WASM_MODE */ +#endif /* !SQLITE_SHELL_WASM_WEB_MODE */ if( z[0]=='-' ){ utf8_printf(stderr, "unknown option: %s\n", z); rc = 1; @@ -9638,7 +9637,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* If a filename is specified, try to open it first */ if( zFN || p->openMode==SHELL_OPEN_HEXDB ){ if( newFlag && zFN && !p->bSafeMode ) shellDeleteFile(zFN); -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( p->bSafeMode && p->openMode!=SHELL_OPEN_HEXDB && zFN @@ -9671,7 +9670,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( (c=='o' && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) @@ -9787,7 +9786,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zFile); }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ if( c=='p' && n>=3 && strncmp(azArg[0], "parameter", n)==0 ){ open_db(p,0); @@ -9957,13 +9956,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){ rc = 2; }else #endif -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){ FILE *inSaved = p->in; int savedLineno = p->lineno; @@ -9998,9 +9997,9 @@ static int do_meta_command(char *zLine, ShellState *p){ p->in = inSaved; p->lineno = savedLineno; }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){ const char *zSrcFile; const char *zDb; @@ -10052,7 +10051,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } close_db(pSrc); }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ @@ -10678,7 +10677,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zSql); }else -#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE) +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_WEB_MODE) if( c=='s' && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) ){ @@ -10699,7 +10698,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zCmd); if( x ) raw_printf(stderr, "System command returns %d\n", x); }else -#endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_WEB_MODE) */ if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ static const char *azBool[] = { "off", "on", "trigger", "full"}; @@ -10879,7 +10878,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(azResult); }else -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE /* Begin redirecting output to the file "testcase-out.txt" */ if( c=='t' && strcmp(azArg[0],"testcase")==0 ){ output_reset(p); @@ -10893,7 +10892,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); } }else -#endif /* !defined(SQLITE_SHELL_WASM_MODE) */ +#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ #ifndef SQLITE_UNTESTABLE if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 ){ @@ -11565,7 +11564,7 @@ static void echo_group_input(ShellState *p, const char *zDo){ if( ShellHasFlag(p, SHFLG_Echo) ) utf8_printf(p->out, "%s\n", zDo); } -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_WASM_WEB_MODE /* ** Alternate one_input_line() impl for wasm mode. This is not in the primary impl ** because we need the global shellState and cannot access it from that function @@ -11596,7 +11595,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ zLine[nZ] = 0; return zLine; } -#endif /* SQLITE_SHELL_WASM_MODE */ +#endif /* SQLITE_SHELL_WASM_WEB_MODE */ /* ** Read input from *in and process it. If *in==0 then input @@ -11979,7 +11978,7 @@ static char *cmdline_option_value(int argc, char **argv, int i){ # endif #endif -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_WASM_WEB_MODE # define main fiddle_main #endif @@ -11993,7 +11992,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ sqlite3_int64 mem_main_enter = sqlite3_memory_used(); #endif char *zErrMsg = 0; -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_WASM_WEB_MODE # define data shellState #else ShellState data; @@ -12013,7 +12012,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ setBinaryMode(stdin, 0); setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_WASM_WEB_MODE stdin_is_interactive = 0; stdout_is_console = 1; #else @@ -12275,7 +12274,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif } data.out = stdout; -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE sqlite3_appendvfs_init(0,0,0); #endif @@ -12543,7 +12542,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ rc = process_input(&data); } } -#ifndef SQLITE_SHELL_WASM_MODE +#ifndef SQLITE_SHELL_WASM_WEB_MODE /* In WASM mode we have to leave the db state in place so that ** client code can "push" SQL into it after this call returns. */ free(azCmd); @@ -12578,12 +12577,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ (unsigned int)(sqlite3_memory_used()-mem_main_enter)); } #endif -#endif /* !SQLITE_SHELL_WASM_MODE */ +#endif /* !SQLITE_SHELL_WASM_WEB_MODE */ return rc; } -#ifdef SQLITE_SHELL_WASM_MODE +#ifdef SQLITE_SHELL_WASM_WEB_MODE /* Only for emcc experimentation purposes. */ int fiddle_experiment(int a,int b){ return a + b; @@ -12704,4 +12703,4 @@ void fiddle_exec(const char * zSql){ memset(&shellState.wasm, 0, sizeof(shellState.wasm)); } } -#endif /* SQLITE_SHELL_WASM_MODE */ +#endif /* SQLITE_SHELL_WASM_WEB_MODE */ From f0af3db7701ba8b5b5a834c3dd0d877efa3a3867 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 12 Jul 2022 10:46:02 +0000 Subject: [PATCH 058/151] Fix a bug in wapptest.tcl introduced by [51255bad4c1fb607]. FossilOrigin-Name: b26d097e099b18ff434467adafe83fc55429488dbb8ce9c879bb03d8c5b7d3eb --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/wapptest.tcl | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index de0503af9a..45fcfefe57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Renamed\sSQLITE_SHELL_WASM_MODE\sto\sSQLITE_SHELL_WASM_WEB_MODE\sand\sno\slonger\sautomatically\senable\sit\sif\s__EMSCRIPTEN__\sis\sdefined,\sin\sorder\sto\sfacilitate\susing\sEmscripten\sto\sbuild\sthe\sshell\sfor\sCLI-based\sWASM\sruntimes\s(which\scannot\smake\suse\sof\sthe\sweb-specific\suser\sinput\schanges).\sThe\sfiddle\sbuild\snow\sexplicitly\spasses\sthe\snew\sflag\son\sat\scompile-time. -D 2022-07-12T09:40:27.255 +C Fix\sa\sbug\sin\swapptest.tcl\sintroduced\sby\s[51255bad4c1fb607]. +D 2022-07-12T10:46:02.702 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1801,7 +1801,7 @@ F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 F test/walvfs.test bccb3e0d235ef85e276f491d34db32c9ada1ea67be8d9f10aabe7b30319ec656 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec -F test/wapptest.tcl 8d69504451f613141fc0107a60c4fdcf59745cb3d421cdcd051deb6cdf36b288 x +F test/wapptest.tcl e3b6d5afa021c39a0f459ea9fbd1077459c1d81fca98eb40af8404ad3fc2360f x F test/where.test d13cd7c24e80009d2b54e2f7a8893c457afa49c64f99359c9eb3fe668ba1d9d4 F test/where2.test 03c21a11e7b90e2845fc3c8b4002fc44cc2797fa74c86ee47d70bd7ea4f29ed6 F test/where3.test 5b4ffc0ac2ea0fe92f02b1244b7531522fe4d7bccf6fa8741d54e82c10e67753 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec96293ead83603ebe5d7f250d6fdc11f22172f05a9513f175331437c3eaa4c8 -R 8020e393ac6ab4de8f603cb939b22712 -U stephan -Z d97d3fda1a826afffbfc7a98df7e436e +P ee059ad5a811a1511e37158f041a7bf9070529d530410d2f1c4395cdd25c6d33 +R 045391b1c16f142da84fbbaaea1c1d59 +U drh +Z 054267434d9e9a8e380eb13bbd806a61 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 924255a5e6..00118e387a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee059ad5a811a1511e37158f041a7bf9070529d530410d2f1c4395cdd25c6d33 \ No newline at end of file +b26d097e099b18ff434467adafe83fc55429488dbb8ce9c879bb03d8c5b7d3eb \ No newline at end of file diff --git a/test/wapptest.tcl b/test/wapptest.tcl index 47eeee5f1d..fa28ec7600 100755 --- a/test/wapptest.tcl +++ b/test/wapptest.tcl @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] wapp.tcl] # G(jobs) - How many sub-processes to run simultaneously. # set G(platform) $::tcl_platform(os)-$::tcl_platform(machine) -set G(cfgglob * +set G(cfgglob) * set G(test) Normal set G(keep) 1 set G(msvc) 0 @@ -31,7 +31,7 @@ set G(stdout) 0 proc wapptest_init {} { global G - set lSave [list platform test keep msvc tcl jobs debug noui stdout] + set lSave [list platform test keep msvc tcl jobs debug noui stdout cfgglob] foreach k $lSave { set A($k) $G($k) } array unset G foreach k $lSave { set G($k) $A($k) } From 8d48e30b397626c09b1e5815011084bc642bf85e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Jul 2022 15:10:16 +0000 Subject: [PATCH 059/151] Update makefiles to fix building the non-amalgamation testfixture with SQLITE_DEBUG. FossilOrigin-Name: d9c4a9d09b6b22d7d95420b495dc7d7a42a0638be5824f6af6630539fe787cd4 --- Makefile.in | 1 + main.mk | 1 + manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Makefile.in b/Makefile.in index 94fa96ea83..526504b069 100644 --- a/Makefile.in +++ b/Makefile.in @@ -503,6 +503,7 @@ TESTSRC2 = \ $(TOP)/src/pcache1.c \ $(TOP)/src/select.c \ $(TOP)/src/tokenize.c \ + $(TOP)/src/treeview.c \ $(TOP)/src/utf.c \ $(TOP)/src/util.c \ $(TOP)/src/vdbeapi.c \ diff --git a/main.mk b/main.mk index 1926a08859..35e42d6ca2 100644 --- a/main.mk +++ b/main.mk @@ -423,6 +423,7 @@ TESTSRC2 = \ $(TOP)/src/select.c \ $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ + $(TOP)/src/treeview.c \ $(TOP)/src/utf.c \ $(TOP)/src/util.c \ $(TOP)/src/vdbeapi.c \ diff --git a/manifest b/manifest index 45fcfefe57..ada57a2c1c 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sa\sbug\sin\swapptest.tcl\sintroduced\sby\s[51255bad4c1fb607]. -D 2022-07-12T10:46:02.702 +C Update\smakefiles\sto\sfix\sbuilding\sthe\snon-amalgamation\stestfixture\swith\sSQLITE_DEBUG. +D 2022-07-12T15:10:16.068 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in ab82a7cf01fd4d2b1164c8cb20513b59f4223a1a6e713938b1fb5b842418020d +F Makefile.in fea7bbb25e4b59bc16631f6ff897acc88525013b62fdeba0a36456ef1548a27f F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc de7cb3e095ce2fdc33513ccd76ebdaeda1483d0ddab0410fe65cbdeadd4c0ee1 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -491,7 +491,7 @@ F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a5412510e5ec952915a7fda34e02079bb4e6ff8f97903f2a3d9ad2dee3e18044 +F main.mk 50e98928657b6d5aeb9445f6130964bd644861cfa049d689db41da317d449517 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ee059ad5a811a1511e37158f041a7bf9070529d530410d2f1c4395cdd25c6d33 -R 045391b1c16f142da84fbbaaea1c1d59 -U drh -Z 054267434d9e9a8e380eb13bbd806a61 +P b26d097e099b18ff434467adafe83fc55429488dbb8ce9c879bb03d8c5b7d3eb +R 22ab8a0d5553115981b624fa454c10d5 +U dan +Z ee54ba27f709e5af1b5cefb61125f247 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 00118e387a..2ee5fcf49b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b26d097e099b18ff434467adafe83fc55429488dbb8ce9c879bb03d8c5b7d3eb \ No newline at end of file +d9c4a9d09b6b22d7d95420b495dc7d7a42a0638be5824f6af6630539fe787cd4 \ No newline at end of file From 03437f7886f048e0275c48137e8d350c82f6c325 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Jul 2022 15:17:50 +0000 Subject: [PATCH 060/151] Fix another test case error message similar to those fixed in [b3d6b3c3]. FossilOrigin-Name: 6d0f677291d2b5ec68c86292da240c5557422aae1290c0844223974449ce539b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fts3fault2.test | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ada57a2c1c..67c01b4f0b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\smakefiles\sto\sfix\sbuilding\sthe\snon-amalgamation\stestfixture\swith\sSQLITE_DEBUG. -D 2022-07-12T15:10:16.068 +C Fix\sanother\stest\scase\serror\smessage\ssimilar\sto\sthose\sfixed\sin\s[b3d6b3c3]. +D 2022-07-12T15:17:50.749 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1026,7 +1026,7 @@ F test/fts3expr4.test f5b2832549f01b1f7f73389fa21d4b875499bc95bf7c8b36271844888c F test/fts3expr5.test a5b9a053becbdb8e973fbf4d6d3abaabeb42d511d1848bd57931f3e0a1cf983e F test/fts3f.test 8c438d5e1cab526b0021988fb1dc70cf3597b006a33ffd6c955ee89929077fe3 F test/fts3fault.test f4e1342acfe6d216a001490e8cd52afac1f9ffe4a11bbcdcb296129a45c5df45 -F test/fts3fault2.test 6a17a11d8034b1c4eca9f3091649273d56c49ff049e2173df8060f94341e9da0 +F test/fts3fault2.test 7b2741e5095367238380b0fcdb837f36c24484c7a5f353659b387df63cf039ec F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3fuzz001.test e3c7b0ce9b04cc02281dcc96812a277f02df03cd7dc082055d87e11eb18aaf56 F test/fts3join.test ee25def5e763ea8879c19e74f862d5191410ccc7259338887a3685e97f512662 @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b26d097e099b18ff434467adafe83fc55429488dbb8ce9c879bb03d8c5b7d3eb -R 22ab8a0d5553115981b624fa454c10d5 +P d9c4a9d09b6b22d7d95420b495dc7d7a42a0638be5824f6af6630539fe787cd4 +R 7e19ab5805b123d32a175181853b886b U dan -Z ee54ba27f709e5af1b5cefb61125f247 +Z 02ab1b6108a49a8c83f37e6697731f43 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2ee5fcf49b..b5e23fb90a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9c4a9d09b6b22d7d95420b495dc7d7a42a0638be5824f6af6630539fe787cd4 \ No newline at end of file +6d0f677291d2b5ec68c86292da240c5557422aae1290c0844223974449ce539b \ No newline at end of file diff --git a/test/fts3fault2.test b/test/fts3fault2.test index 5c182d73c9..913f9baa05 100644 --- a/test/fts3fault2.test +++ b/test/fts3fault2.test @@ -215,12 +215,15 @@ do_faultsim_test 8.1 -faults oom* -prep { faultsim_test_result {0 {}} } +set ::TMPDBERROR [list 1 \ + {unable to open a temporary database file for storing temporary tables} +] do_faultsim_test 8.2 -faults oom* -prep { faultsim_restore_and_reopen } -body { execsql { ALTER TABLE t8 RENAME TO t8ii } } -test { - faultsim_test_result {0 {}} + faultsim_test_result {0 {}} $::TMPDBERROR } #------------------------------------------------------------------------- From 4413ec72557c00b2c3c33e5910edff4250148a7a Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 12 Jul 2022 15:53:02 +0000 Subject: [PATCH 061/151] Renamed the SQLITE_SHELL_WASM_WEB_MODE to SQLITE_SHELL_FIDDLE, which seems to be more in line with project convensions and indicates that that flag is only intended for /fiddle mode, as opposed to arbitrary wasm-on-the-web use. FossilOrigin-Name: d1d019bfa2f62b0dc39bba42e17786ca2e4362b6d11d206e5445a051a0f93ae0 --- Makefile.in | 2 +- manifest | 16 +++--- manifest.uuid | 2 +- src/shell.c.in | 141 ++++++++++++++++++++++++------------------------- 4 files changed, 80 insertions(+), 81 deletions(-) diff --git a/Makefile.in b/Makefile.in index 526504b069..9a2ed9a183 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1541,7 +1541,7 @@ $(fiddle_module_js): Makefile sqlite3.c shell.c \ emcc -o $@ $(emcc_flags) \ -sEXPORT_NAME=initFiddleModule \ -sEXPORTED_FUNCTIONS=@$(fiddle_dir_abs)/EXPORTED_FUNCTIONS.fiddle \ - -DSQLITE_SHELL_WASM_WEB_MODE \ + -DSQLITE_SHELL_FIDDLE \ sqlite3.c shell.c gzip < $@ > $@.gz gzip < $(fiddle_dir)/fiddle-module.wasm > $(fiddle_dir)/fiddle-module.wasm.gz diff --git a/manifest b/manifest index 67c01b4f0b..90baf71089 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sanother\stest\scase\serror\smessage\ssimilar\sto\sthose\sfixed\sin\s[b3d6b3c3]. -D 2022-07-12T15:17:50.749 +C Renamed\sthe\sSQLITE_SHELL_WASM_WEB_MODE\sto\sSQLITE_SHELL_FIDDLE,\swhich\sseems\sto\sbe\smore\sin\sline\swith\sproject\sconvensions\sand\sindicates\sthat\sthat\sflag\sis\sonly\sintended\sfor\s/fiddle\smode,\sas\sopposed\sto\sarbitrary\swasm-on-the-web\suse. +D 2022-07-12T15:53:02.771 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in fea7bbb25e4b59bc16631f6ff897acc88525013b62fdeba0a36456ef1548a27f +F Makefile.in 002f746b3e065e843055a23b8032b93763e68ccbaafa2b8f68532d0dfbe68deb F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc de7cb3e095ce2fdc33513ccd76ebdaeda1483d0ddab0410fe65cbdeadd4c0ee1 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -571,7 +571,7 @@ F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c 84a8443e3723e908730d754f54df4e1dacc1eb7073c0bd79c9d5efe977a9f5b9 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 061b628dd1d3025a4ef3278e7128ce148d5f6bf58b8c1173c3c9e118d16198c6 -F src/shell.c.in b928ddd7c3b5c63a9b625695e8af6a8831efd267d5187b9418f9eca05157bda3 +F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 F src/sqlite.h.in 01573eae96721f2a8ee2a9e3b7140ceeba2e9c44350911890b89b8ff0dcf6781 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9c4a9d09b6b22d7d95420b495dc7d7a42a0638be5824f6af6630539fe787cd4 -R 7e19ab5805b123d32a175181853b886b -U dan -Z 02ab1b6108a49a8c83f37e6697731f43 +P 6d0f677291d2b5ec68c86292da240c5557422aae1290c0844223974449ce539b +R f399d27350e58dc1d29ac8e998e2c75b +U stephan +Z edf4e8f892024032a392fd9df52c0e15 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b5e23fb90a..74644f03e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d0f677291d2b5ec68c86292da240c5557422aae1290c0844223974449ce539b \ No newline at end of file +d1d019bfa2f62b0dc39bba42e17786ca2e4362b6d11d206e5445a051a0f93ae0 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index bc5e889a59..641a93840d 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -38,13 +38,12 @@ #endif /* -** Use -DSQLITE_SHELL_WASM_WEB_MODE to build for Emscripten targeting -** web browser use. We do not automatically do that using -** ifdef(__EMSCRIPTEN__) because it should be possible to build the -** shell with Emscripten for use in shell-based WASM/WASI runtime -** environments. The browser-mode build has much different user input -** requirements and the "wasm web" build rewires the user input -** subsystem to account for that. +** If SQLITE_SHELL_FIDDLE is defined then the shell is modified +** somewhat for use as a WASM module in a web browser. This flag +** should only be used when building the "fiddle" web application, as +** the browser-mode build has much different user input requirements +** and this build mode rewires the user input subsystem to account for +** that. */ /* @@ -700,7 +699,7 @@ static char *local_getline(char *zLine, FILE *in){ ** be freed by the caller or else passed back into this routine via the ** zPrior argument for reuse. */ -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ char *zPrompt; char *zResult; @@ -720,7 +719,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ } return zResult; } -#endif /* !SQLITE_SHELL_WASM_WEB_MODE */ +#endif /* !SQLITE_SHELL_FIDDLE */ /* ** Return the value of a hexadecimal digit. Return -1 if the input @@ -1026,7 +1025,7 @@ INCLUDE ../ext/misc/decimal.c INCLUDE ../ext/misc/ieee754.c INCLUDE ../ext/misc/series.c INCLUDE ../ext/misc/regexp.c -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE INCLUDE ../ext/misc/fileio.c INCLUDE ../ext/misc/completion.c INCLUDE ../ext/misc/appendvfs.c @@ -1161,7 +1160,7 @@ struct ShellState { char *zNonce; /* Nonce for temporary safe-mode excapes */ EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ -#ifdef SQLITE_SHELL_WASM_WEB_MODE +#ifdef SQLITE_SHELL_FIDDLE struct { const char * zInput; /* Input string from wasm/JS proxy */ const char * zPos; /* Cursor pos into zInput */ @@ -1169,7 +1168,7 @@ struct ShellState { #endif }; -#ifdef SQLITE_SHELL_WASM_WEB_MODE +#ifdef SQLITE_SHELL_FIDDLE static ShellState shellState; #endif @@ -1837,7 +1836,7 @@ static int safeModeAuth( UNUSED_PARAMETER(zA4); switch( op ){ case SQLITE_ATTACH: { -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE /* In WASM builds the filesystem is a virtual sandbox, so ** there's no harm in using ATTACH. */ failIfSafeMode(p, "cannot run ATTACH in safe mode"); @@ -4251,7 +4250,7 @@ static int run_schema_dump_query( */ static const char *(azHelp[]) = { #if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) \ - && !defined(SQLITE_SHELL_WASM_WEB_MODE) + && !defined(SQLITE_SHELL_FIDDLE) ".archive ... Manage SQL archives", " Each command must have exactly one of the following options:", " -c, --create Create a new archive", @@ -4277,7 +4276,7 @@ static const char *(azHelp[]) = { #ifndef SQLITE_OMIT_AUTHORIZATION ".auth ON|OFF Show authorizer callbacks", #endif -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".backup ?DB? FILE Backup DB (default \"main\") to FILE", " Options:", " --append Use the appendvfs", @@ -4285,11 +4284,11 @@ static const char *(azHelp[]) = { #endif ".bail on|off Stop after hitting an error. Default OFF", ".binary on|off Turn binary output on or off. Default OFF", -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".cd DIRECTORY Change the working directory to DIRECTORY", #endif ".changes on|off Show number of rows changed by SQL", -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".check GLOB Fail if output since .testcase does not match", ".clone NEWDB Clone data into NEWDB from the existing database", #endif @@ -4315,11 +4314,11 @@ static const char *(azHelp[]) = { " trace Like \"full\" but enable \"PRAGMA vdbe_trace\"", #endif " trigger Like \"full\" but also show trigger bytecode", -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".excel Display the output of next command in spreadsheet", " --bom Put a UTF8 byte-order mark on intermediate file", #endif -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".exit ?CODE? Exit this program with return-code CODE", #endif ".expert EXPERIMENTAL. Suggest indexes for queries", @@ -4330,7 +4329,7 @@ static const char *(azHelp[]) = { ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", ".headers on|off Turn display of headers on or off", ".help ?-all? ?PATTERN? Show help text for PATTERN", -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".import FILE TABLE Import data from FILE into TABLE", " Options:", " --ascii Use \\037 and \\036 as column and row separators", @@ -4359,10 +4358,10 @@ static const char *(azHelp[]) = { ".lint OPTIONS Report potential schema issues.", " Options:", " fkey-indexes Find missing foreign key indexes", -#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_WASM_WEB_MODE) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE) ".load FILE ?ENTRY? Load an extension library", #endif -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".log FILE|off Turn logging on or off. FILE can be stderr/stdout", #endif ".mode MODE ?OPTIONS? Set output mode", @@ -4389,11 +4388,11 @@ static const char *(azHelp[]) = { " --quote Quote output text as SQL literals", " --noquote Do not quote output text", " TABLE The name of SQL table used for \"insert\" mode", -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".nonce STRING Suspend safe mode for one command if nonce matches", #endif ".nullvalue STRING Use STRING in place of NULL values", -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", " If FILE begins with '|' then open as a pipe", " --bom Put a UTF8 byte-order mark at the beginning", @@ -4415,7 +4414,7 @@ static const char *(azHelp[]) = { " --nofollow Do not follow symbolic links", " --readonly Open FILE readonly", " --zip FILE is a ZIP archive", -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".output ?FILE? Send output to FILE or stdout if FILE is omitted", " If FILE begins with '|' then open it as a pipe.", " Options:", @@ -4439,7 +4438,7 @@ static const char *(azHelp[]) = { " --reset Reset the count for each input and interrupt", #endif ".prompt MAIN CONTINUE Replace the standard prompts", -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".quit Exit this program", ".read FILE Read input from FILE or command output", " If FILE begins with \"|\", it is a command that generates the input.", @@ -4452,7 +4451,7 @@ static const char *(azHelp[]) = { " --no-rowids Do not attempt to recover rowid values", " that are not also INTEGER PRIMARY KEYs", #endif -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", ".save ?OPTIONS? FILE Write database to FILE (an alias for .backup ...)", #endif @@ -4489,7 +4488,7 @@ static const char *(azHelp[]) = { " --sha3-384 Use the sha3-384 algorithm", " --sha3-512 Use the sha3-512 algorithm", " Any other argument is a LIKE pattern for tables to hash", -#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_WEB_MODE) +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) ".shell CMD ARGS... Run CMD ARGS... in a system shell", #endif ".show Show the current values for various settings", @@ -4498,11 +4497,11 @@ static const char *(azHelp[]) = { " on Turn on automatic stat display", " stmt Show statement stats", " vmstep Show the virtual machine step count only", -#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_WEB_MODE) +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) ".system CMD ARGS... Run CMD ARGS... in a system shell", #endif ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE ".testcase NAME Begin redirecting output to 'testcase-out.txt'", #endif ".testctrl CMD ... Run various sqlite3_test_control() operations", @@ -5055,7 +5054,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_regexp_init(p->db, 0, 0); sqlite3_ieee_init(p->db, 0, 0); sqlite3_series_init(p->db, 0, 0); -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE sqlite3_fileio_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); #endif @@ -8185,7 +8184,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) \ - && !defined(SQLITE_SHELL_WASM_WEB_MODE) + && !defined(SQLITE_SHELL_FIDDLE) if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){ open_db(p, 0); failIfSafeMode(p, "cannot run .archive in safe mode"); @@ -8193,7 +8192,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0) || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0) ){ @@ -8262,7 +8261,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } close_db(pDest); }else -#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 ){ if( nArg==2 ){ @@ -8293,7 +8292,7 @@ static int do_meta_command(char *zLine, ShellState *p){ test_breakpoint(); }else -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( c=='c' && strcmp(azArg[0],"cd")==0 ){ failIfSafeMode(p, "cannot run .cd in safe mode"); if( nArg==2 ){ @@ -8313,7 +8312,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } }else -#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){ if( nArg==2 ){ @@ -8324,7 +8323,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE /* Cancel output redirection, if it is currently set (by .testcase) ** Then read the content of the testcase-out.txt file and compare against ** azArg[1]. If there are differences, report an error and exit. @@ -8349,9 +8348,9 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zRes); }else -#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){ failIfSafeMode(p, "cannot run .clone in safe mode"); if( nArg==2 ){ @@ -8361,7 +8360,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } }else -#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ if( c=='c' && strncmp(azArg[0], "connection", n)==0 ){ if( nArg==1 ){ @@ -8650,7 +8649,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); rc = 2; @@ -8910,7 +8909,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( c=='i' && strncmp(azArg[0], "import", n)==0 ){ char *zTable = 0; /* Insert data into this table */ char *zSchema = 0; /* within this schema (may default to "main") */ @@ -9201,7 +9200,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sCtx.nRow, sCtx.nErr, sCtx.nLine-1); } }else -#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){ @@ -9391,7 +9390,7 @@ static int do_meta_command(char *zLine, ShellState *p){ lintDotCommand(p, azArg, nArg); }else -#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_WASM_WEB_MODE) +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE) if( c=='l' && strncmp(azArg[0], "load", n)==0 ){ const char *zFile, *zProc; char *zErrMsg = 0; @@ -9413,7 +9412,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( c=='l' && strncmp(azArg[0], "log", n)==0 ){ failIfSafeMode(p, "cannot run .log in safe mode"); if( nArg!=2 ){ @@ -9550,7 +9549,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->cMode = p->mode; }else -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( c=='n' && strcmp(azArg[0], "nonce")==0 ){ if( nArg!=2 ){ raw_printf(stderr, "Usage: .nonce NONCE\n"); @@ -9565,7 +9564,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** at the end of this procedure */ } }else -#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 ){ if( nArg==2 ){ @@ -9587,7 +9586,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* Check for command-line arguments */ for(iName=1; iNameszMax = integerValue(azArg[++iName]); #endif /* SQLITE_OMIT_DESERIALIZE */ }else -#endif /* !SQLITE_SHELL_WASM_WEB_MODE */ +#endif /* !SQLITE_SHELL_FIDDLE */ if( z[0]=='-' ){ utf8_printf(stderr, "unknown option: %s\n", z); rc = 1; @@ -9637,7 +9636,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* If a filename is specified, try to open it first */ if( zFN || p->openMode==SHELL_OPEN_HEXDB ){ if( newFlag && zFN && !p->bSafeMode ) shellDeleteFile(zFN); -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( p->bSafeMode && p->openMode!=SHELL_OPEN_HEXDB && zFN @@ -9670,7 +9669,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( (c=='o' && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) @@ -9786,7 +9785,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zFile); }else -#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ if( c=='p' && n>=3 && strncmp(azArg[0], "parameter", n)==0 ){ open_db(p,0); @@ -9956,13 +9955,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){ rc = 2; }else #endif -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){ FILE *inSaved = p->in; int savedLineno = p->lineno; @@ -9997,9 +9996,9 @@ static int do_meta_command(char *zLine, ShellState *p){ p->in = inSaved; p->lineno = savedLineno; }else -#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){ const char *zSrcFile; const char *zDb; @@ -10051,7 +10050,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } close_db(pSrc); }else -#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ @@ -10677,7 +10676,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zSql); }else -#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_WEB_MODE) +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) if( c=='s' && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) ){ @@ -10698,7 +10697,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zCmd); if( x ) raw_printf(stderr, "System command returns %d\n", x); }else -#endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */ if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ static const char *azBool[] = { "off", "on", "trigger", "full"}; @@ -10878,7 +10877,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(azResult); }else -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE /* Begin redirecting output to the file "testcase-out.txt" */ if( c=='t' && strcmp(azArg[0],"testcase")==0 ){ output_reset(p); @@ -10892,7 +10891,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); } }else -#endif /* !defined(SQLITE_SHELL_WASM_WEB_MODE) */ +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 ){ @@ -11564,7 +11563,7 @@ static void echo_group_input(ShellState *p, const char *zDo){ if( ShellHasFlag(p, SHFLG_Echo) ) utf8_printf(p->out, "%s\n", zDo); } -#ifdef SQLITE_SHELL_WASM_WEB_MODE +#ifdef SQLITE_SHELL_FIDDLE /* ** Alternate one_input_line() impl for wasm mode. This is not in the primary impl ** because we need the global shellState and cannot access it from that function @@ -11595,7 +11594,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ zLine[nZ] = 0; return zLine; } -#endif /* SQLITE_SHELL_WASM_WEB_MODE */ +#endif /* SQLITE_SHELL_FIDDLE */ /* ** Read input from *in and process it. If *in==0 then input @@ -11978,7 +11977,7 @@ static char *cmdline_option_value(int argc, char **argv, int i){ # endif #endif -#ifdef SQLITE_SHELL_WASM_WEB_MODE +#ifdef SQLITE_SHELL_FIDDLE # define main fiddle_main #endif @@ -11992,7 +11991,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ sqlite3_int64 mem_main_enter = sqlite3_memory_used(); #endif char *zErrMsg = 0; -#ifdef SQLITE_SHELL_WASM_WEB_MODE +#ifdef SQLITE_SHELL_FIDDLE # define data shellState #else ShellState data; @@ -12012,7 +12011,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ setBinaryMode(stdin, 0); setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ -#ifdef SQLITE_SHELL_WASM_WEB_MODE +#ifdef SQLITE_SHELL_FIDDLE stdin_is_interactive = 0; stdout_is_console = 1; #else @@ -12274,7 +12273,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif } data.out = stdout; -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE sqlite3_appendvfs_init(0,0,0); #endif @@ -12542,7 +12541,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ rc = process_input(&data); } } -#ifndef SQLITE_SHELL_WASM_WEB_MODE +#ifndef SQLITE_SHELL_FIDDLE /* In WASM mode we have to leave the db state in place so that ** client code can "push" SQL into it after this call returns. */ free(azCmd); @@ -12577,12 +12576,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ (unsigned int)(sqlite3_memory_used()-mem_main_enter)); } #endif -#endif /* !SQLITE_SHELL_WASM_WEB_MODE */ +#endif /* !SQLITE_SHELL_FIDDLE */ return rc; } -#ifdef SQLITE_SHELL_WASM_WEB_MODE +#ifdef SQLITE_SHELL_FIDDLE /* Only for emcc experimentation purposes. */ int fiddle_experiment(int a,int b){ return a + b; @@ -12703,4 +12702,4 @@ void fiddle_exec(const char * zSql){ memset(&shellState.wasm, 0, sizeof(shellState.wasm)); } } -#endif /* SQLITE_SHELL_WASM_WEB_MODE */ +#endif /* SQLITE_SHELL_FIDDLE */ From 0505851451d7495feecc8f0386c80c3e7863c03e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Jul 2022 20:31:16 +0000 Subject: [PATCH 062/151] Add test/testrunner.tcl, an experimental script for distributing the work of veryquick.test between multiple processes. FossilOrigin-Name: ef229cbb7ffbeb8c8877dff70e9d6d43050d2297dee582a37df3a0caaebd2a41 --- manifest | 14 +- manifest.uuid | 2 +- test/testrunner.tcl | 456 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 466 insertions(+), 6 deletions(-) create mode 100644 test/testrunner.tcl diff --git a/manifest b/manifest index 67c01b4f0b..6bf93332c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\stest\scase\serror\smessage\ssimilar\sto\sthose\sfixed\sin\s[b3d6b3c3]. -D 2022-07-12T15:17:50.749 +C Add\stest/testrunner.tcl,\san\sexperimental\sscript\sfor\sdistributing\sthe\swork\sof\sveryquick.test\sbetween\smultiple\sprocesses. +D 2022-07-12T20:31:16.301 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1507,6 +1507,7 @@ F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 +F test/testrunner.tcl bfaaddd58df6176af83159e3b27767c53abd87db68619a300234ad7fbf9aeed1 F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1979,8 +1980,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9c4a9d09b6b22d7d95420b495dc7d7a42a0638be5824f6af6630539fe787cd4 -R 7e19ab5805b123d32a175181853b886b +P 6d0f677291d2b5ec68c86292da240c5557422aae1290c0844223974449ce539b +R 8212f2952574d93e95a3664f57ee334d +T *branch * testrunner +T *sym-testrunner * +T -sym-trunk * U dan -Z 02ab1b6108a49a8c83f37e6697731f43 +Z a723c28ab5e45345241e356c4c65467f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b5e23fb90a..0e8e50979b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d0f677291d2b5ec68c86292da240c5557422aae1290c0844223974449ce539b \ No newline at end of file +ef229cbb7ffbeb8c8877dff70e9d6d43050d2297dee582a37df3a0caaebd2a41 \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl new file mode 100644 index 0000000000..7ebbbb712f --- /dev/null +++ b/test/testrunner.tcl @@ -0,0 +1,456 @@ + + +#------------------------------------------------------------------------- +# Usage: +# +proc usage {} { + puts stderr "Usage: $::argv0 ?SWITCHES?" + puts stderr "" + puts stderr "where SWITCHES are:" + puts stderr " --jobs NUMBER-OF-JOBS" + exit 1 +} +#------------------------------------------------------------------------- + + +switch -- [lindex $argv 0] { + helper { + set R(helper) 1 + set R(helper_id) [lindex $argv 1] + set argv [list --testdir=testdir$R(helper_id)] + } + + default { + set R(helper) 0 + set R(helper_id) 0 + } +} + + +set R(dbname) [file normalize testrunner.db] +set R(logname) [file normalize testrunner.log] +set R(timeout) 10000 +set R(nHelper) 4 +set R(info_script) [file normalize [info script]] + +if {$R(helper)==0} { + for {set ii 0} {$ii < [llength $argv]} {incr ii} { + set a [lindex $argv $ii] + set n [string length $a] + + if {($n>2 && [string match "$a*" --jobs]) || $a=="-j"} { + incr ii + set R(nHelper) [lindex $argv $ii] + } else { + usage + } + } + + set argv [list] +} + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +db close + +# The database schema used by the testset database. +# +set R(schema) { + DROP TABLE IF EXISTS script; + DROP TABLE IF EXISTS msg; + + CREATE TABLE script( + filename TEXT PRIMARY KEY, -- full path to test script + state TEXT CHECK( state IN ('ready', 'running', 'done') ), + testfixtureid, -- Id of process that ran script + nerr INTEGER, -- if 'done', the number of errors + ntest INTEGER, -- if 'done', the number of tests + output TEXT -- full output of test script + ); + + CREATE TABLE msg( + id INTEGER PRIMARY KEY, + msg TEXT + ); +} + + +#-------------------------------------------------------------------- +# This is temporary! +# +# Return a list of all scripts in the "veryquick" test. +# +proc all_veryquick_scripts {} { + set OMIT { + async2.test async3.test backup_ioerr.test corrupt.test + corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test + crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test + fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test + misc7.test mutex2.test notify2.test onefile.test pagerfault2.test + savepoint4.test savepoint6.test select9.test + speed1.test speed1p.test speed2.test speed3.test speed4.test + speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test + thread003.test thread004.test thread005.test trans2.test vacuum3.test + incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test + vtab_err.test walslow.test walcrash.test walcrash3.test + walthread.test rtree3.test indexfault.test securedel2.test + sort3.test sort4.test fts4growth.test fts4growth2.test + bigsort.test walprotocol.test mmap4.test fuzzer2.test + walcrash2.test e_fkey.test backup.test + writecrash.test + + fts4merge.test fts4merge2.test fts4merge4.test fts4check.test + fts4merge5.test + fts3cov.test fts3snippet.test fts3corrupt2.test fts3an.test + fts3defer.test fts4langid.test fts3sort.test fts5unicode.test + rtree4.test sessionbig.test + + all.test async.test quick.test veryquick.test + memleak.test permutations.test soak.test fts3.test + mallocAll.test rtree.test full.test extraquick.test + session.test rbu.test + } + + set testdir [file normalize $::testdir] + set ret [list] + + foreach f [glob -nocomplain $testdir/*.test] { + if {[lsearch $OMIT [file tail $f]]<0 + && [string match *malloc* $f]==0 + && [string match *ioerr* $f]==0 + && [string match *fault* $f]==0 + && [string match *bigfile* $f]==0 + && [string match *_err* $f]==0 + && [string match *fts5corrupt* $f]==0 + && [string match *fts5big* $f]==0 + && [string match *fts5aj* $f]==0 + } { + lappend ret $f + } + } + + set ret +} +#proc all_veryquick_scripts {} { +# set testdir [file normalize $::testdir] +# glob -nocomplain $testdir/select*.test +#} +#-------------------------------------------------------------------- + + +proc make_new_testset {} { + global R + + sqlite3 db $R(dbname) + db eval $R(schema) + foreach s [all_veryquick_scripts] { + db eval { INSERT INTO script(filename, state) VALUES ($s, 'ready') } + } + + # db eval { SELECT filename FROM Script ORDER BY 1 } { puts $filename } + # exit + + + db close +} + +proc get_next_test {} { + global R + set myid $R(helper_id) + + sqlite3 db $R(dbname) + db timeout $R(timeout) + db eval { BEGIN EXCLUSIVE } + set f [db one { + SELECT filename FROM script WHERE state='ready' ORDER BY 1 LIMIT 1 + }] + if {$f!=""} { + db eval { + UPDATE script SET state='running', testfixtureid=$myid WHERE filename=$f + } + } + db eval { COMMIT } + db close + + return $f +} + +proc r_write_db {tcl} { + global R + sqlite3 db $R(dbname) + db timeout $R(timeout) + db eval { BEGIN EXCLUSIVE } + uplevel $tcl + db eval { COMMIT } + db close +} + +proc r_read_db {tcl} { + global R + sqlite3 db $R(dbname) + db timeout $R(timeout) + db eval { BEGIN } + uplevel $tcl + db eval { COMMIT } + db close +} + +proc r_set_test_result {filename ms nerr ntest output} { + global R + + set f [file tail $filename] + if {$nerr==0} { + set msg "$f... Ok" + } else { + set msg "$f... FAILED - $nerr errors of $ntest tests" + } + append msg " (${ms}ms)" + if {$R(helper)} { + append msg " (helper $R(helper_id))" + } + + r_write_db { + db eval { + UPDATE script + SET state='done', output=$output, nerr=$nerr, ntest=$ntest + WHERE filename=$filename; + + INSERT INTO msg(msg) VALUES ($msg); + } + } +} + +set R(iNextMsg) 1 +proc r_get_messages {{db ""}} { + global R + + if {$db==""} { + sqlite3 rgmhandle $R(dbname) + set dbhandle rgmhandle + $dbhandle timeout $R(timeout) + } else { + set dbhandle $db + } + + $dbhandle transaction { + set next $R(iNextMsg) + set ret [$dbhandle eval {SELECT msg FROM msg WHERE id>=$next}] + set R(iNextMsg) [$dbhandle one {SELECT COALESCE(max(id), 0)+1 FROM msg}] + } + + if {$db==""} { + rgmhandle close + } + + set ret +} + +#-------------------------------------------------------------------------- +# + + +set ::R_INSTALL_PUTS_WRAPPER { + proc puts_sts_wrapper {args} { + set n [llength $args] + if {$n==1 || ($n==2 && [string first [lindex $args 0] -nonewline]==0)} { + uplevel puts_into_caller $args + } else { + # A channel was explicitly specified. + uplevel puts_sts_original $args + } + } + rename puts puts_sts_original + proc puts {args} { uplevel puts_sts_wrapper $args } +} + +proc r_install_puts_wrapper {} $::R_INSTALL_PUTS_WRAPPER +proc r_uninstall_puts_wrapper {} { + rename puts "" + rename puts_sts_original puts +} + +proc slave_test_script {script} { + + # Create the interpreter used to run the test script. + interp create tinterp + + # Populate some global variables that tester.tcl expects to see. + foreach {var value} [list \ + ::argv0 $::argv0 \ + ::argv {} \ + ::SLAVE 1 \ + ] { + interp eval tinterp [list set $var $value] + } + + # The alias used to access the global test counters. + tinterp alias set_test_counter set_test_counter + + # Set up an empty ::cmdlinearg array in the slave. + interp eval tinterp [list array set ::cmdlinearg [array get ::cmdlinearg]] + + # Set up the ::G array in the slave. + interp eval tinterp [list array set ::G [array get ::G]] + interp eval tinterp [list set ::G(runner.tcl) 1] + + interp eval tinterp $::R_INSTALL_PUTS_WRAPPER + tinterp alias puts_into_caller puts_into_caller + + # Load the various test interfaces implemented in C. + load_testfixture_extensions tinterp + + # Run the test script. + set rc [catch { interp eval tinterp $script } msg opt] + if {$rc} { + puts_into_caller $msg + puts_into_caller [dict get $opt -errorinfo] + incr ::TC(errors) + } + + # Check if the interpreter call [run_thread_tests] + if { [interp eval tinterp {info exists ::run_thread_tests_called}] } { + set ::run_thread_tests_called 1 + } + + # Delete the interpreter used to run the test script. + interp delete tinterp +} + +proc slave_test_file {zFile} { + set tail [file tail $zFile] + + # Remember the value of the shared-cache setting. So that it is possible + # to check afterwards that it was not modified by the test script. + # + ifcapable shared_cache { set scs [sqlite3_enable_shared_cache] } + + # Run the test script in a slave interpreter. + # + unset -nocomplain ::run_thread_tests_called + reset_prng_state + set ::sqlite_open_file_count 0 + set time [time { slave_test_script [list source $zFile] }] + set ms [expr [lindex $time 0] / 1000] + + r_install_puts_wrapper + + # Test that all files opened by the test script were closed. Omit this + # if the test script has "thread" in its name. The open file counter + # is not thread-safe. + # + if {[info exists ::run_thread_tests_called]==0} { + do_test ${tail}-closeallfiles { expr {$::sqlite_open_file_count>0} } {0} + } + set ::sqlite_open_file_count 0 + + # Test that the global "shared-cache" setting was not altered by + # the test script. + # + ifcapable shared_cache { + set res [expr {[sqlite3_enable_shared_cache] == $scs}] + do_test ${tail}-sharedcachesetting [list set {} $res] 1 + } + + # Add some info to the output. + # + output2 "Time: $tail $ms ms" + show_memstats + + r_uninstall_puts_wrapper + return $ms +} + +proc puts_into_caller {args} { + global R + if {[llength $args]==1} { + append R(output) [lindex $args 0] + append R(output) "\n" + } else { + append R(output) [lindex $args 1] + } +} + +set R(nHelperRunning) 0 +if {$R(helper)==0} { + cd $cmdlinearg(TESTFIXTURE_HOME) + make_new_testset + for {set ii 1} {$ii <= $R(nHelper)} {incr ii} { + set cmd "[info nameofexec] $R(info_script) helper $ii 2>@1" + puts "Launching helper $ii ($cmd)" + set chan [open "|$cmd" r] + fconfigure $chan -blocking false + fileevent $chan readable [list r_helper_readable $ii $chan] + incr R(nHelperRunning) + } + cd $cmdlinearg(testdir) +} + +proc r_helper_readable {id chan} { + puts "helper $id:[gets $chan]" + if {[eof $chan]} { + puts "helper $id is FINISHED" + incr ::R(nHelperRunning) -1 + close $chan + } +} + +if {$R(helper) || $R(nHelper)<4} { + while { ""!=[set f [get_next_test]] } { + set R(output) "" + set TC(count) 0 + set TC(errors) 0 + set ms [slave_test_file $f] + r_set_test_result $f $ms $TC(errors) $TC(count) $R(output) + + if {$R(helper)==0} { + foreach msg [r_get_messages] { puts $msg } + } + } +} + +set TTT 0 +sqlite3 db $R(dbname) +db timeout $R(timeout) +while {$R(nHelperRunning)>0} { + after 250 { incr TTT } + vwait TTT + foreach msg [r_get_messages db] { puts $msg } +} +db close + +set errcode 0 +if {$R(helper)==0} { + sqlite3 db $R(dbname) + db timeout $R(timeout) + db eval { SELECT sum(nerr) AS nerr, sum(ntest) AS ntest FROM script } { + puts "$nerr errors from $ntest tests." + } + if {$nerr>0} { + db eval { SELECT filename FROM script WHERE nerr>0 } { + lappend errlist [file tail $filename] + } + puts "Errors in: $errlist" + set errcode 1 + } + + set errlist [list] + db eval { SELECT filename FROM script WHERE state!='done' } { + lappend errlist [file tail $filename] + } + if {$errlist!=[list]} { + puts "Tests DID NOT FINISH (crashed?): $errlist" + set errcode 1 + } + + set fd [open $R(logname) w] + db eval {SELECT output FROM script ORDER BY filename} { + puts $fd $output + } + close $fd + + puts "Test database is $R(dbname)" + puts "Test log file is $R(logname)" +} + +exit $errcode + From b88bf865c31b89ce2265864a79d2815c624b5ba2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 13 Jul 2022 15:52:15 +0000 Subject: [PATCH 063/151] The query flattener should not run if the subquery is a compound that contains a RIGHT JOIN in any arm and the subquery is not the first element of the outer query. Otherwise, prior elements of the outer query will not have the JT_LTORJ flag set. Fix for the problem reported in [forum:/forumpost/174afeae5734d42d|forum post 174afeae5734d42d]. FossilOrigin-Name: 274e244c85935084b2f0f85176283f018bf9b74e7703f985bd5a2f6f8bdcff5d --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 19 ++++++++++++++++--- test/join8.test | 15 +++++++++++++++ 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 90baf71089..efe77e8f88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Renamed\sthe\sSQLITE_SHELL_WASM_WEB_MODE\sto\sSQLITE_SHELL_FIDDLE,\swhich\sseems\sto\sbe\smore\sin\sline\swith\sproject\sconvensions\sand\sindicates\sthat\sthat\sflag\sis\sonly\sintended\sfor\s/fiddle\smode,\sas\sopposed\sto\sarbitrary\swasm-on-the-web\suse. -D 2022-07-12T15:53:02.771 +C The\squery\sflattener\sshould\snot\srun\sif\sthe\ssubquery\sis\sa\scompound\sthat\scontains\na\sRIGHT\sJOIN\sin\sany\sarm\sand\sthe\ssubquery\sis\snot\sthe\sfirst\selement\sof\sthe\nouter\squery.\s\sOtherwise,\sprior\selements\sof\sthe\souter\squery\swill\snot\shave\nthe\sJT_LTORJ\sflag\sset.\s\sFix\sfor\sthe\sproblem\sreported\sin\n[forum:/forumpost/174afeae5734d42d|forum\spost\s174afeae5734d42d]. +D 2022-07-13T15:52:15.028 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -570,7 +570,7 @@ F src/printf.c 6166a30417b05c5b2f82e1f183f75faa2926ad60531c0b688a57dbc951441a20 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c 84a8443e3723e908730d754f54df4e1dacc1eb7073c0bd79c9d5efe977a9f5b9 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 061b628dd1d3025a4ef3278e7128ce148d5f6bf58b8c1173c3c9e118d16198c6 +F src/select.c 01e5099d4fc15f84a38e3b439b27cdb2d62e4bb138cbcaf835dd46386f8fd82c F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 F src/sqlite.h.in 01573eae96721f2a8ee2a9e3b7140ceeba2e9c44350911890b89b8ff0dcf6781 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1169,7 +1169,7 @@ F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test d22b6cba8fb59ab3f1c82701434c360705eb12d4ce200c449f37b018fc47681a F test/join6.test f809c025fa253f9e150c0e9afd4cef8813257bceeb6f46e04041228c9403cc2c F test/join7.test 2268dcbb54b724391dda3748ea95c60d960607ffeed67885675998e7117697f6 -F test/join8.test d0e75bf880a1b1ee8fda57d8187170c5b2ff0c0cd0e1d6d474faa99fe1283d8d +F test/join8.test ef5fb09a7ce6b59addb8bd16e11607db6c44a0afcac5774a5dc03193fd0a1df5 F test/join9.test 9056ddd3b0c0f4f9d658f4521038d9a37dc23ead8ca9a505d0b0db2b6a471e05 F test/joinA.test 7eab225dc1c1ab258a5e62513a4ed7cabbd3db971d59d5d92f4fb6fa14c12f6a F test/joinB.test 1b2ba3fc8568b49411787fccbf540570c148e9b6a53a30f80691cb6268098ded @@ -1979,8 +1979,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d0f677291d2b5ec68c86292da240c5557422aae1290c0844223974449ce539b -R f399d27350e58dc1d29ac8e998e2c75b -U stephan -Z edf4e8f892024032a392fd9df52c0e15 +P d1d019bfa2f62b0dc39bba42e17786ca2e4362b6d11d206e5445a051a0f93ae0 +R 8ba9f41e04ffe4b4f2fce5afccce75ed +U drh +Z 64b6a3b4244651c63a98e29aa670c738 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 74644f03e7..29d537888d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d1d019bfa2f62b0dc39bba42e17786ca2e4362b6d11d206e5445a051a0f93ae0 \ No newline at end of file +274e244c85935084b2f0f85176283f018bf9b74e7703f985bd5a2f6f8bdcff5d \ No newline at end of file diff --git a/src/select.c b/src/select.c index 133283b1f1..d712cc3ee6 100644 --- a/src/select.c +++ b/src/select.c @@ -4120,6 +4120,9 @@ static void renumberCursors( ** (17d2) DISTINCT ** (17e) the subquery may not contain window functions, and ** (17f) the subquery must not be the RHS of a LEFT JOIN. +** (17g) either the subquery is the first element of the outer +** query or there are no RIGHT or FULL JOINs in any arm +** of the subquery. (This is a duplicate of condition (27b).) ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -4171,7 +4174,11 @@ static void renumberCursors( ** See also (3) for restrictions on LEFT JOIN. ** ** (27) The subquery may not contain a FULL or RIGHT JOIN unless it -** is the first element of the parent query. +** is the first element of the parent query. This must be the +** the case if: +** (27a) the subquery is not compound query, and +** (27b) the subquery is a compound query and the RIGHT JOIN occurs +** in any arm of the compound query. (See also (17g).) ** ** (28) The subquery is not a MATERIALIZED CTE. ** @@ -4296,7 +4303,7 @@ static int flattenSubquery( assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */ if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ - return 0; /* Restriction (27) */ + return 0; /* Restriction (27a) */ } if( pSubitem->fg.isCte && pSubitem->u2.pCteUse->eM10d==M10d_Yes ){ return 0; /* (28) */ @@ -4316,7 +4323,7 @@ static int flattenSubquery( ** NATURAL join or a join that as an ON or USING clause. ** ** These conditions are sufficient to keep an EP_OuterON from being - ** flattened into an EP_InnerON. Restrictions (3a) and (27) prevent + ** flattened into an EP_InnerON. Restrictions (3a) and (27a) prevent ** an EP_InnerON from being flattened into an EP_OuterON. */ if( pSubSrc->nSrc>=2 @@ -4358,6 +4365,12 @@ static int flattenSubquery( ){ return 0; } + if( iFrom>0 && (pSub1->pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + /* Without this restriction, the JT_LTORJ flag would end up being + ** omitted on left-hand tables of the right join that is being + ** flattened. */ + return 0; /* Restrictions (17g), (27b) */ + } testcase( pSub1->pSrc->nSrc>1 ); } diff --git a/test/join8.test b/test/join8.test index 97b0fe13df..481430556c 100644 --- a/test/join8.test +++ b/test/join8.test @@ -786,4 +786,19 @@ do_execsql_test join8-25020 { SELECT 1 FROM t1 LEFT JOIN t2 ON true JOIN t3 ON (b2 IN (a1)) FULL JOIN t4 ON true; } {1} +# 2022-07-13 +# forum/forumpost/174afeae57 +# +reset_db +db null - +do_execsql_test join8-26000 { + CREATE TABLE t1(a INT); + CREATE TABLE t2(b INT, c INT); + CREATE VIEW t3(d) AS SELECT NULL FROM t2 FULL OUTER JOIN t1 ON c=a UNION ALL SELECT b FROM t2; + INSERT INTO t1(a) VALUES (NULL); + INSERT INTO t2(b, c) VALUES (99, NULL); + SELECT DISTINCT b, c, d FROM t2, t3 WHERE b<>0 + UNION SELECT DISTINCT b, c, d FROM t2, t3 WHERE b ISNULL; +} {99 - - 99 - 99} + finish_test From e9a7ebe19f7010b5b05d87451a0d4e835ecf7871 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 13 Jul 2022 17:46:42 +0000 Subject: [PATCH 064/151] Fix testrunner.tcl so that it checks for memory leaks. FossilOrigin-Name: 106f6724d54ccec3edf8c9a0422b89c4f227adb26021ed6f0fc91392ef4b3fc5 --- manifest | 15 +- manifest.uuid | 2 +- test/testrunner.tcl | 371 +++++++++++++++++++++++++++++--------------- 3 files changed, 251 insertions(+), 137 deletions(-) diff --git a/manifest b/manifest index 6bf93332c7..0bd8756440 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest/testrunner.tcl,\san\sexperimental\sscript\sfor\sdistributing\sthe\swork\sof\sveryquick.test\sbetween\smultiple\sprocesses. -D 2022-07-12T20:31:16.301 +C Fix\stestrunner.tcl\sso\sthat\sit\schecks\sfor\smemory\sleaks. +D 2022-07-13T17:46:42.435 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1507,7 +1507,7 @@ F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 -F test/testrunner.tcl bfaaddd58df6176af83159e3b27767c53abd87db68619a300234ad7fbf9aeed1 +F test/testrunner.tcl 3bdd2d32319c65f34d1aafe6fe66aac4881e78ded4880719f48203aeea13b1c4 F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1980,11 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d0f677291d2b5ec68c86292da240c5557422aae1290c0844223974449ce539b -R 8212f2952574d93e95a3664f57ee334d -T *branch * testrunner -T *sym-testrunner * -T -sym-trunk * +P ef229cbb7ffbeb8c8877dff70e9d6d43050d2297dee582a37df3a0caaebd2a41 +R 9ae28163af03b5806c7e690857875476 U dan -Z a723c28ab5e45345241e356c4c65467f +Z 14e1b9ab4edb92095656b55f758ae601 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0e8e50979b..38cc6028ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef229cbb7ffbeb8c8877dff70e9d6d43050d2297dee582a37df3a0caaebd2a41 \ No newline at end of file +106f6724d54ccec3edf8c9a0422b89c4f227adb26021ed6f0fc91392ef4b3fc5 \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 7ebbbb712f..ce99a3a09f 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -12,7 +12,90 @@ proc usage {} { } #------------------------------------------------------------------------- +#------------------------------------------------------------------------- +# The database schema used by the testrunner.db database. +# +set R(schema) { + DROP TABLE IF EXISTS script; + DROP TABLE IF EXISTS msg; + DROP TABLE IF EXISTS malloc; + CREATE TABLE script( + filename TEXT PRIMARY KEY, -- full path to test script + state TEXT CHECK( state IN ('ready', 'running', 'done') ), + testfixtureid, -- Id of process that ran script + time INTEGER, -- Time in ms + nerr INTEGER, -- if 'done', the number of errors + ntest INTEGER, -- if 'done', the number of tests + output TEXT -- full output of test script + ); + + CREATE TABLE malloc( + id INTEGER PRIMARY KEY, + nmalloc INTEGER, + nbyte INTEGER, + leaker TEXT + ); + + CREATE TABLE msg( + id INTEGER PRIMARY KEY, + msg TEXT + ); +} +#------------------------------------------------------------------------- + +#------------------------------------------------------------------------- +# Try to estimate a the number of processes to use. +# +# Command [guess_number_of_cores] attempts to glean the number of logical +# cores. Command [default_njob] returns the default value for the --jobs +# switch. +# +proc guess_number_of_cores {} { + set ret 4 + catch { + set fd [open "|nproc" r] + set ret [gets $fd] + close $fd + set ret [expr $ret] + } + return $ret +} + +proc default_njob {} { + set nCore [guess_number_of_cores] + set nHelper [expr int($nCore*0.75)] + expr $nHelper>0 ? $nHelper : 1 +} +#------------------------------------------------------------------------- + + + +set R(dbname) [file normalize testrunner.db] +set R(logname) [file normalize testrunner.log] +set R(info_script) [file normalize [info script]] +set R(timeout) 10000 ;# Default busy-timeout for testrunner. +set R(nJob) [default_njob] ;# Default number of helper processes +set R(leaker) "" ;# Name of first script to leak memory + + +# Parse the command line options. There are two ways to invoke this +# script - to create a helper or coordinator process. If there are +# no helper processes, the coordinator runs test scripts. +# +# To create a helper process: +# +# testrunner.tcl helper ID +# +# where ID is an integer greater than 0. The process will create and +# run tests in the "testdir$ID" directory. Helper processes are only +# created by coordinators - there is no need for a user to create +# helper processes manually. +# +# If the first argument is anything other than "helper", then a coordinator +# process is started. See the implementation of the [usage] proc above for +# details. +# switch -- [lindex $argv 0] { helper { set R(helper) 1 @@ -23,16 +106,9 @@ switch -- [lindex $argv 0] { default { set R(helper) 0 set R(helper_id) 0 + } } - - -set R(dbname) [file normalize testrunner.db] -set R(logname) [file normalize testrunner.log] -set R(timeout) 10000 -set R(nHelper) 4 -set R(info_script) [file normalize [info script]] - if {$R(helper)==0} { for {set ii 0} {$ii < [llength $argv]} {incr ii} { set a [lindex $argv $ii] @@ -40,7 +116,7 @@ if {$R(helper)==0} { if {($n>2 && [string match "$a*" --jobs]) || $a=="-j"} { incr ii - set R(nHelper) [lindex $argv $ii] + set R(nJob) [lindex $argv $ii] } else { usage } @@ -53,28 +129,6 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl db close -# The database schema used by the testset database. -# -set R(schema) { - DROP TABLE IF EXISTS script; - DROP TABLE IF EXISTS msg; - - CREATE TABLE script( - filename TEXT PRIMARY KEY, -- full path to test script - state TEXT CHECK( state IN ('ready', 'running', 'done') ), - testfixtureid, -- Id of process that ran script - nerr INTEGER, -- if 'done', the number of errors - ntest INTEGER, -- if 'done', the number of tests - output TEXT -- full output of test script - ); - - CREATE TABLE msg( - id INTEGER PRIMARY KEY, - msg TEXT - ); -} - - #-------------------------------------------------------------------- # This is temporary! # @@ -109,6 +163,7 @@ proc all_veryquick_scripts {} { memleak.test permutations.test soak.test fts3.test mallocAll.test rtree.test full.test extraquick.test session.test rbu.test + } set testdir [file normalize $::testdir] @@ -131,68 +186,48 @@ proc all_veryquick_scripts {} { set ret } -#proc all_veryquick_scripts {} { -# set testdir [file normalize $::testdir] -# glob -nocomplain $testdir/select*.test -#} #-------------------------------------------------------------------- -proc make_new_testset {} { - global R - - sqlite3 db $R(dbname) - db eval $R(schema) - foreach s [all_veryquick_scripts] { - db eval { INSERT INTO script(filename, state) VALUES ($s, 'ready') } - } - - # db eval { SELECT filename FROM Script ORDER BY 1 } { puts $filename } - # exit - - - db close -} - -proc get_next_test {} { - global R - set myid $R(helper_id) - - sqlite3 db $R(dbname) - db timeout $R(timeout) - db eval { BEGIN EXCLUSIVE } - set f [db one { - SELECT filename FROM script WHERE state='ready' ORDER BY 1 LIMIT 1 - }] - if {$f!=""} { - db eval { - UPDATE script SET state='running', testfixtureid=$myid WHERE filename=$f - } - } - db eval { COMMIT } - db close - - return $f -} - proc r_write_db {tcl} { global R sqlite3 db $R(dbname) db timeout $R(timeout) db eval { BEGIN EXCLUSIVE } + uplevel $tcl + db eval { COMMIT } db close } -proc r_read_db {tcl} { +proc make_new_testset {} { global R - sqlite3 db $R(dbname) - db timeout $R(timeout) - db eval { BEGIN } - uplevel $tcl - db eval { COMMIT } - db close + + r_write_db { + db eval $R(schema) + foreach s [all_veryquick_scripts] { + db eval { INSERT INTO script(filename, state) VALUES ($s, 'ready') } + } + } +} + +proc get_next_test {} { + global R + set myid $R(helper_id) + + r_write_db { + set f [db one { + SELECT filename FROM script WHERE state='ready' ORDER BY 1 LIMIT 1 + }] + if {$f!=""} { + db eval { + UPDATE script SET state='running', testfixtureid=$myid WHERE filename=$f + } + } + } + + return $f } proc r_set_test_result {filename ms nerr ntest output} { @@ -209,10 +244,17 @@ proc r_set_test_result {filename ms nerr ntest output} { append msg " (helper $R(helper_id))" } + sqlite3_shutdown + set nMalloc [lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1] + set nByte [sqlite3_memory_used] + if {($nByte>0 || $nMalloc>0) && $R(leaker)==""} { + set R(leaker) $filename + } + r_write_db { db eval { UPDATE script - SET state='done', output=$output, nerr=$nerr, ntest=$ntest + SET state='done', output=$output, nerr=$nerr, ntest=$ntest, time=$ms WHERE filename=$filename; INSERT INTO msg(msg) VALUES ($msg); @@ -245,10 +287,30 @@ proc r_get_messages {{db ""}} { set ret } +# This is called after all tests have been run to write the leaked memory +# report into the malloc table of testrunner.db. +# +proc r_memory_report {} { + global R + + sqlite3_shutdown + + set nMalloc [lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1] + set nByte [sqlite3_memory_used] + set id $R(helper_id) + set leaker $R(leaker) + + r_write_db { + db eval { + INSERT INTO malloc(id, nMalloc, nByte, leaker) + VALUES($id, $nMalloc, $nByte, $leaker) + } + } +} + + #-------------------------------------------------------------------------- # - - set ::R_INSTALL_PUTS_WRAPPER { proc puts_sts_wrapper {args} { set n [llength $args] @@ -370,11 +432,88 @@ proc puts_into_caller {args} { } } -set R(nHelperRunning) 0 +#------------------------------------------------------------------------- +# +proc r_final_report {} { + global R + + sqlite3 db $R(dbname) + db timeout $R(timeout) + + set errcode 0 + + # Create the text log file. This is just the concatenation of the + # 'output' column of the database for every script that was run. + set fd [open $R(logname) w] + db eval {SELECT output FROM script ORDER BY filename} { + puts $fd $output + } + close $fd + + # Check if any scripts reported errors. If so, print one line noting + # how many errors, and another identifying the scripts in which they + # occured. Or, if no errors occurred, print out "no errors at all!". + sqlite3 db $R(dbname) + db timeout $R(timeout) + db eval { SELECT sum(nerr) AS nerr, sum(ntest) AS ntest FROM script } { } + puts "$nerr errors from $ntest tests." + if {$nerr>0} { + db eval { SELECT filename FROM script WHERE nerr>0 } { + lappend errlist [file tail $filename] + } + puts "Errors in: $errlist" + set errcode 1 + } + + # Check if any scripts were not run or did not finish. Print out a + # line identifying them if there are any. + set errlist [list] + db eval { SELECT filename FROM script WHERE state!='done' } { + lappend errlist [file tail $filename] + } + if {$errlist!=[list]} { + puts "Tests DID NOT FINISH (crashed?): $errlist" + set errcode 1 + } + + set bLeak 0 + db eval { + SELECT id, nmalloc, nbyte, leaker FROM malloc + WHERE nmalloc>0 OR nbyte>0 + } { + if {$id==0} { + set line "This process " + } else { + set line "Helper $id " + } + append line "leaked $nbyte byte in $nmalloc allocations" + if {$leaker!=""} { append line " (perhaps in [file tail $leaker])" } + puts $line + set bLeak 1 + } + if {$bLeak==0} { + puts "No leaks - all allocations freed." + } + + db close + + puts "Test database is $R(dbname)" + puts "Test log file is $R(logname)" + if {$errcode} { + puts "This test has FAILED." + } + return $errcode +} + + if {$R(helper)==0} { - cd $cmdlinearg(TESTFIXTURE_HOME) make_new_testset - for {set ii 1} {$ii <= $R(nHelper)} {incr ii} { +} + +set R(nHelperRunning) 0 +if {$R(helper)==0 && $R(nJob)>1} { + cd $cmdlinearg(TESTFIXTURE_HOME) + for {set ii 1} {$ii <= $R(nJob)} {incr ii} { set cmd "[info nameofexec] $R(info_script) helper $ii 2>@1" puts "Launching helper $ii ($cmd)" set chan [open "|$cmd" r] @@ -386,70 +525,48 @@ if {$R(helper)==0} { } proc r_helper_readable {id chan} { - puts "helper $id:[gets $chan]" + set data [gets $chan] + if {$data!=""} { puts "helper $id:[gets $chan]" } if {[eof $chan]} { - puts "helper $id is FINISHED" + puts "helper $id is finished" incr ::R(nHelperRunning) -1 close $chan } } -if {$R(helper) || $R(nHelper)<4} { +if {$R(nHelperRunning)==0} { while { ""!=[set f [get_next_test]] } { set R(output) "" set TC(count) 0 set TC(errors) 0 set ms [slave_test_file $f] + r_set_test_result $f $ms $TC(errors) $TC(count) $R(output) if {$R(helper)==0} { foreach msg [r_get_messages] { puts $msg } } } -} -set TTT 0 -sqlite3 db $R(dbname) -db timeout $R(timeout) -while {$R(nHelperRunning)>0} { - after 250 { incr TTT } - vwait TTT - foreach msg [r_get_messages db] { puts $msg } + # Tests are finished - write a record into testrunner.db describing + # any memory leaks. + r_memory_report + +} else { + set TTT 0 + sqlite3 db $R(dbname) + db timeout $R(timeout) + while {$R(nHelperRunning)>0} { + after 250 { incr TTT } + vwait TTT + foreach msg [r_get_messages db] { puts $msg } + } + db close } -db close set errcode 0 if {$R(helper)==0} { - sqlite3 db $R(dbname) - db timeout $R(timeout) - db eval { SELECT sum(nerr) AS nerr, sum(ntest) AS ntest FROM script } { - puts "$nerr errors from $ntest tests." - } - if {$nerr>0} { - db eval { SELECT filename FROM script WHERE nerr>0 } { - lappend errlist [file tail $filename] - } - puts "Errors in: $errlist" - set errcode 1 - } - - set errlist [list] - db eval { SELECT filename FROM script WHERE state!='done' } { - lappend errlist [file tail $filename] - } - if {$errlist!=[list]} { - puts "Tests DID NOT FINISH (crashed?): $errlist" - set errcode 1 - } - - set fd [open $R(logname) w] - db eval {SELECT output FROM script ORDER BY filename} { - puts $fd $output - } - close $fd - - puts "Test database is $R(dbname)" - puts "Test log file is $R(logname)" + set errcode [r_final_report] } exit $errcode From 95cc6a5e1137af8c46e7534ddb55e39d9dffb612 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 13 Jul 2022 19:57:35 +0000 Subject: [PATCH 065/151] Update testrunner.tcl to allow the user to specify which tests to run on the command line. FossilOrigin-Name: 900febcf362fa5f592c640d16177f33c13aab11ce31a61c7e18ff1be6e70bf9b --- manifest | 13 ++--- manifest.uuid | 2 +- test/testrunner.tcl | 82 ++++++------------------------ test/testset.tcl | 120 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 74 deletions(-) create mode 100644 test/testset.tcl diff --git a/manifest b/manifest index 0bd8756440..5589f9eeee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stestrunner.tcl\sso\sthat\sit\schecks\sfor\smemory\sleaks. -D 2022-07-13T17:46:42.435 +C Update\stestrunner.tcl\sto\sallow\sthe\suser\sto\sspecify\swhich\stests\sto\srun\son\sthe\scommand\sline. +D 2022-07-13T19:57:35.922 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1507,7 +1507,8 @@ F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 -F test/testrunner.tcl 3bdd2d32319c65f34d1aafe6fe66aac4881e78ded4880719f48203aeea13b1c4 +F test/testrunner.tcl 297f066a0c2c78e552a29060701825fcfd16cd915c19d51654112d54df2feacc +F test/testset.tcl 8295bbfe2da21bc32a818b55b074d572a24a44fccfb74da919194a749bb7654c F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1980,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef229cbb7ffbeb8c8877dff70e9d6d43050d2297dee582a37df3a0caaebd2a41 -R 9ae28163af03b5806c7e690857875476 +P 106f6724d54ccec3edf8c9a0422b89c4f227adb26021ed6f0fc91392ef4b3fc5 +R 032a0dd6c8d21e7f2f5ea3dc860bfe74 U dan -Z 14e1b9ab4edb92095656b55f758ae601 +Z f52161ed1f80342c1b5b733a625a7b6b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 38cc6028ad..cda73bf94a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -106f6724d54ccec3edf8c9a0422b89c4f227adb26021ed6f0fc91392ef4b3fc5 \ No newline at end of file +900febcf362fa5f592c640d16177f33c13aab11ce31a61c7e18ff1be6e70bf9b \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl index ce99a3a09f..f3863f2023 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -4,7 +4,7 @@ # Usage: # proc usage {} { - puts stderr "Usage: $::argv0 ?SWITCHES?" + puts stderr "Usage: $::argv0 ?SWITCHES? ?PATTERN? ..." puts stderr "" puts stderr "where SWITCHES are:" puts stderr " --jobs NUMBER-OF-JOBS" @@ -70,7 +70,6 @@ proc default_njob {} { #------------------------------------------------------------------------- - set R(dbname) [file normalize testrunner.db] set R(logname) [file normalize testrunner.log] set R(info_script) [file normalize [info script]] @@ -78,6 +77,10 @@ set R(timeout) 10000 ;# Default busy-timeout for testrunner. set R(nJob) [default_njob] ;# Default number of helper processes set R(leaker) "" ;# Name of first script to leak memory +set R(patternlist) [list] + +set testdir [file dirname $argv0] +source $testdir/testset.tcl # Parse the command line options. There are two ways to invoke this # script - to create a helper or coordinator process. If there are @@ -114,80 +117,24 @@ if {$R(helper)==0} { set a [lindex $argv $ii] set n [string length $a] - if {($n>2 && [string match "$a*" --jobs]) || $a=="-j"} { - incr ii - set R(nJob) [lindex $argv $ii] + if {[string range $a 0 0]=="-"} { + if {($n>2 && [string match "$a*" --jobs]) || $a=="-j"} { + incr ii + set R(nJob) [lindex $argv $ii] + } else { + usage + } } else { - usage + lappend R(patternlist) [string map {% * _ .} $a] } } set argv [list] } -set testdir [file dirname $argv0] source $testdir/tester.tcl db close -#-------------------------------------------------------------------- -# This is temporary! -# -# Return a list of all scripts in the "veryquick" test. -# -proc all_veryquick_scripts {} { - set OMIT { - async2.test async3.test backup_ioerr.test corrupt.test - corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test - crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test - fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test - misc7.test mutex2.test notify2.test onefile.test pagerfault2.test - savepoint4.test savepoint6.test select9.test - speed1.test speed1p.test speed2.test speed3.test speed4.test - speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test - thread003.test thread004.test thread005.test trans2.test vacuum3.test - incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test - vtab_err.test walslow.test walcrash.test walcrash3.test - walthread.test rtree3.test indexfault.test securedel2.test - sort3.test sort4.test fts4growth.test fts4growth2.test - bigsort.test walprotocol.test mmap4.test fuzzer2.test - walcrash2.test e_fkey.test backup.test - writecrash.test - - fts4merge.test fts4merge2.test fts4merge4.test fts4check.test - fts4merge5.test - fts3cov.test fts3snippet.test fts3corrupt2.test fts3an.test - fts3defer.test fts4langid.test fts3sort.test fts5unicode.test - rtree4.test sessionbig.test - - all.test async.test quick.test veryquick.test - memleak.test permutations.test soak.test fts3.test - mallocAll.test rtree.test full.test extraquick.test - session.test rbu.test - - } - - set testdir [file normalize $::testdir] - set ret [list] - - foreach f [glob -nocomplain $testdir/*.test] { - if {[lsearch $OMIT [file tail $f]]<0 - && [string match *malloc* $f]==0 - && [string match *ioerr* $f]==0 - && [string match *fault* $f]==0 - && [string match *bigfile* $f]==0 - && [string match *_err* $f]==0 - && [string match *fts5corrupt* $f]==0 - && [string match *fts5big* $f]==0 - && [string match *fts5aj* $f]==0 - } { - lappend ret $f - } - } - - set ret -} -#-------------------------------------------------------------------- - proc r_write_db {tcl} { global R @@ -204,9 +151,10 @@ proc r_write_db {tcl} { proc make_new_testset {} { global R + set scripts [testset_patternlist $R(patternlist)] r_write_db { db eval $R(schema) - foreach s [all_veryquick_scripts] { + foreach s $scripts { db eval { INSERT INTO script(filename, state) VALUES ($s, 'ready') } } } diff --git a/test/testset.tcl b/test/testset.tcl new file mode 100644 index 0000000000..b98f2e2a5a --- /dev/null +++ b/test/testset.tcl @@ -0,0 +1,120 @@ + + +# Commands in this file: +# +# testset_all +# Return a list of all test scripts designed to be run individually. +# +# testset_veryquick +# The subset of [testset_all] meant to run as veryquick.test. +# + +set D(testdir) [file dir [file normalize [info script]]] + +proc testset_all {} { + global D + set ret [list] + + # The following tests are driver scripts that themselves run lots of other + # test scripts. They should be ignored here. + set drivers { + all.test async.test quick.test veryquick.test + memleak.test permutations.test soak.test fts3.test + mallocAll.test rtree.test full.test extraquick.test + session.test rbu.test + } + + set srcdir [file dirname $D(testdir)] + set ret [glob -nocomplain \ + $srcdir/test/*.test \ + $srcdir/ext/rtree/*.test \ + $srcdir/ext/fts5/test/*.test \ + $srcdir/ext/expert/*.test \ + $srcdir/ext/session/*.test \ + ] + set ret [ts_filter $ret $drivers] + return $ret +} + +proc testset_veryquick {} { + set ret [testset_all] + + set ret [ts_filter $ret { + async2.test async3.test backup_ioerr.test corrupt.test + corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test + crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test + fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test + misc7.test mutex2.test notify2.test onefile.test pagerfault2.test + savepoint4.test savepoint6.test select9.test + speed1.test speed1p.test speed2.test speed3.test speed4.test + speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test + thread003.test thread004.test thread005.test trans2.test vacuum3.test + incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test + vtab_err.test walslow.test walcrash.test walcrash3.test + walthread.test rtree3.test indexfault.test securedel2.test + sort3.test sort4.test fts4growth.test fts4growth2.test + bigsort.test walprotocol.test mmap4.test fuzzer2.test + walcrash2.test e_fkey.test backup.test + + fts4merge.test fts4merge2.test fts4merge4.test fts4check.test + fts4merge5.test + fts3cov.test fts3snippet.test fts3corrupt2.test fts3an.test + fts3defer.test fts4langid.test fts3sort.test fts5unicode.test + + rtree4.test + sessionbig.test + }] + + set ret [ts_filter $ret { + *malloc* *ioerr* *fault* *bigfile* *_err* *fts5corrupt* *fts5big* *fts5aj* + }] + + return $ret +} + +proc ts_filter {input exlist} { + foreach f $input { set a($f) 1 } + foreach e $exlist { array unset a */$e } + array names a +} + +proc testset_patternlist {patternlist} { + set nPat [llength $patternlist] + + if {$nPat==0} { + set scripts [testset_veryquick] + } else { + set ii 0 + set p0 [lindex $patternlist 0] + + if {$p0=="veryquick"} { + set scripts [testset_veryquick] + incr ii + } elseif {$p0=="all"} { + set scripts [testset_all] + incr ii + } else { + set scripts [testset_all] + } + + if {$nPat>$ii} { + array set S [list] + foreach f $scripts { set a([file tail $f]) $f } + + foreach p [lrange $patternlist $ii end] { + set nList [llength [array names a $p]] + if {$nList==0} { + puts stderr "Argument $p matches no scripts (typo?)" + exit 1 + } + foreach n [array names a $p] { set S($a($n)) 1 } + } + } + + set scripts [lsort [array names S]] + } + + set scripts +} + + From cda165ff9a716ea2fa0eb024566f94b15fb746bc Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 13 Jul 2022 20:26:37 +0000 Subject: [PATCH 066/151] Fix a problem preventing "testrunnter.tcl all" from working. FossilOrigin-Name: 0ed1e83c6fc12acd06ecf7210a869bebaf5e5e75762e5f16bf1834ecab717d59 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testset.tcl | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5589f9eeee..e22ffa00e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stestrunner.tcl\sto\sallow\sthe\suser\sto\sspecify\swhich\stests\sto\srun\son\sthe\scommand\sline. -D 2022-07-13T19:57:35.922 +C Fix\sa\sproblem\spreventing\s"testrunnter.tcl\sall"\sfrom\sworking. +D 2022-07-13T20:26:37.216 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1508,7 +1508,7 @@ F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 F test/testrunner.tcl 297f066a0c2c78e552a29060701825fcfd16cd915c19d51654112d54df2feacc -F test/testset.tcl 8295bbfe2da21bc32a818b55b074d572a24a44fccfb74da919194a749bb7654c +F test/testset.tcl 27a6bbbc93cbbcf442c57e3c023e6b5d7304dc415e09eb0e9ac61edd0e6c1fbe F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 106f6724d54ccec3edf8c9a0422b89c4f227adb26021ed6f0fc91392ef4b3fc5 -R 032a0dd6c8d21e7f2f5ea3dc860bfe74 +P 900febcf362fa5f592c640d16177f33c13aab11ce31a61c7e18ff1be6e70bf9b +R 0ad59c4f151fdc0de195dce40012b69c U dan -Z f52161ed1f80342c1b5b733a625a7b6b +Z f4f6164f8a169ec2d085a00d87e67f68 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cda73bf94a..025c54bb03 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -900febcf362fa5f592c640d16177f33c13aab11ce31a61c7e18ff1be6e70bf9b \ No newline at end of file +0ed1e83c6fc12acd06ecf7210a869bebaf5e5e75762e5f16bf1834ecab717d59 \ No newline at end of file diff --git a/test/testset.tcl b/test/testset.tcl index b98f2e2a5a..8f761d686a 100644 --- a/test/testset.tcl +++ b/test/testset.tcl @@ -109,9 +109,10 @@ proc testset_patternlist {patternlist} { } foreach n [array names a $p] { set S($a($n)) 1 } } + + set scripts [lsort [array names S]] } - set scripts [lsort [array names S]] } set scripts From 4f28ee9d7609947b5a1cce53d5badaa025f9371a Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 13 Jul 2022 21:10:11 +0000 Subject: [PATCH 067/151] Minor change to help message in testrunner.tcl. FossilOrigin-Name: e4f9cb01d790c27394cc021e7d4a16f4d5feda2d2493abd82e9660d7e6e9fec0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 44e348e1cf..93534498fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sscript\stest/testrunner.tcl.\sFor\srunning\sa\sset\sof\stest\sscripts\susing\nmultiple\sprocesses. -D 2022-07-13T21:02:07.599 +C Minor\schange\sto\shelp\smessage\sin\stestrunner.tcl. +D 2022-07-13T21:10:11.168 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1507,7 +1507,7 @@ F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 -F test/testrunner.tcl 39e43ba90d8fe2d2694049af5ac53861c04a42600d3a8ccb1d07eaaaf350c806 +F test/testrunner.tcl 10b2f2c9c029dc611c6a34215edf29174397370b5c9e2df7a40e5560933bffe4 F test/testset.tcl 27a6bbbc93cbbcf442c57e3c023e6b5d7304dc415e09eb0e9ac61edd0e6c1fbe F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 274e244c85935084b2f0f85176283f018bf9b74e7703f985bd5a2f6f8bdcff5d 0ed1e83c6fc12acd06ecf7210a869bebaf5e5e75762e5f16bf1834ecab717d59 -R 6e5b00a7e5284e8563026668673ce8cf +P 0122e93dc19c228546908b9ef5c58f88d27d79233523e2d09a4bbd56b0c492f9 +R 7e269f5dfdef1a9282f328f55b519c5c U dan -Z 363eb180a41ae46020bac6b29842c080 +Z 246b3b776865c8a91afa511145e5ca7c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5b94b4fbde..5ff7c89e01 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0122e93dc19c228546908b9ef5c58f88d27d79233523e2d09a4bbd56b0c492f9 \ No newline at end of file +e4f9cb01d790c27394cc021e7d4a16f4d5feda2d2493abd82e9660d7e6e9fec0 \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 9d04b252b0..dfdc0a5960 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -6,7 +6,7 @@ proc usage {} { set a0 testrunner.tcl - puts stderr "Usage: $a0 ?SWITCHES? ?PATTERN? ..." + puts stderr "Usage: $a0 ?SWITCHES? ?all|veryquick? ?PATTERNS?" puts stderr "" puts stderr "where SWITCHES are:" puts stderr " --jobs NUMBER-OF-JOBS" From 615aeceaffa37185a4d5656261669024fbab010e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 13 Jul 2022 21:28:19 +0000 Subject: [PATCH 068/151] Fix testrunner.tcl so that it can detect the number of logical cores on osx. FossilOrigin-Name: 14918f28221a3124b78a490fbb483279551ccc5a0032ea854ff0ac365684cc60 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 93534498fb..4f0f666584 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schange\sto\shelp\smessage\sin\stestrunner.tcl. -D 2022-07-13T21:10:11.168 +C Fix\stestrunner.tcl\sso\sthat\sit\scan\sdetect\sthe\snumber\sof\slogical\scores\son\sosx. +D 2022-07-13T21:28:19.394 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1507,7 +1507,7 @@ F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 -F test/testrunner.tcl 10b2f2c9c029dc611c6a34215edf29174397370b5c9e2df7a40e5560933bffe4 +F test/testrunner.tcl da340a87fd8b5c94e0d2196b086b96a90de0ce16bfdb71c1254d543bcee1614a F test/testset.tcl 27a6bbbc93cbbcf442c57e3c023e6b5d7304dc415e09eb0e9ac61edd0e6c1fbe F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0122e93dc19c228546908b9ef5c58f88d27d79233523e2d09a4bbd56b0c492f9 -R 7e269f5dfdef1a9282f328f55b519c5c +P e4f9cb01d790c27394cc021e7d4a16f4d5feda2d2493abd82e9660d7e6e9fec0 +R 9a0d40a26ff73e3e7c23af9cf6fe1abf U dan -Z 246b3b776865c8a91afa511145e5ca7c +Z 0c6e6152833b38a594a27d8bd483ee08 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5ff7c89e01..b38d313dcb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4f9cb01d790c27394cc021e7d4a16f4d5feda2d2493abd82e9660d7e6e9fec0 \ No newline at end of file +14918f28221a3124b78a490fbb483279551ccc5a0032ea854ff0ac365684cc60 \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl index dfdc0a5960..568c599136 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -62,8 +62,14 @@ set R(schema) { # proc guess_number_of_cores {} { set ret 4 + + if {$::tcl_platform(os)=="Darwin"} { + set cmd "sysctl -n hw.logicalcpu" + } else { + set cmd "nproc" + } catch { - set fd [open "|nproc" r] + set fd [open "|$cmd" r] set ret [gets $fd] close $fd set ret [expr $ret] From e834484d892b2d39671ad78e9fda9b774660810c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 14 Jul 2022 01:48:27 +0000 Subject: [PATCH 069/151] When applying the omit-ORDER-BY optimization, defer deleting the AST of the deleted ORDER BY clause until after code generation ends. FossilOrigin-Name: f22f95b838873f1d2a320afe3d0f4e4847948fcd343097b93a9f684a6f66d6ba --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4f0f666584..a3df518eea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stestrunner.tcl\sso\sthat\sit\scan\sdetect\sthe\snumber\sof\slogical\scores\son\sosx. -D 2022-07-13T21:28:19.394 +C When\sapplying\sthe\somit-ORDER-BY\soptimization,\sdefer\sdeleting\sthe\sAST\sof\nthe\sdeleted\sORDER\sBY\sclause\suntil\safter\scode\sgeneration\sends. +D 2022-07-14T01:48:27.069 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -570,7 +570,7 @@ F src/printf.c 6166a30417b05c5b2f82e1f183f75faa2926ad60531c0b688a57dbc951441a20 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c 84a8443e3723e908730d754f54df4e1dacc1eb7073c0bd79c9d5efe977a9f5b9 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 01e5099d4fc15f84a38e3b439b27cdb2d62e4bb138cbcaf835dd46386f8fd82c +F src/select.c 45b93eee3c349f46240ddc14344365bbf34579ec332bd4c7bc061945e38172e2 F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 F src/sqlite.h.in 01573eae96721f2a8ee2a9e3b7140ceeba2e9c44350911890b89b8ff0dcf6781 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e4f9cb01d790c27394cc021e7d4a16f4d5feda2d2493abd82e9660d7e6e9fec0 -R 9a0d40a26ff73e3e7c23af9cf6fe1abf -U dan -Z 0c6e6152833b38a594a27d8bd483ee08 +P 14918f28221a3124b78a490fbb483279551ccc5a0032ea854ff0ac365684cc60 +R d2c6af8b43cf81f08214974216426964 +U drh +Z 663440ae7ea06bca4d8f20ca5b3783d1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b38d313dcb..4924411865 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14918f28221a3124b78a490fbb483279551ccc5a0032ea854ff0ac365684cc60 \ No newline at end of file +f22f95b838873f1d2a320afe3d0f4e4847948fcd343097b93a9f684a6f66d6ba \ No newline at end of file diff --git a/src/select.c b/src/select.c index d712cc3ee6..90ba47fa8a 100644 --- a/src/select.c +++ b/src/select.c @@ -6834,7 +6834,9 @@ int sqlite3Select( ){ SELECTTRACE(0x100,pParse,p, ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); - sqlite3ExprListDelete(db, pSub->pOrderBy); + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3ExprListDelete, + pSub->pOrderBy); pSub->pOrderBy = 0; } From ca0720a9d80b232b3b52481c0ae081e78daf4133 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Jul 2022 18:09:56 +0000 Subject: [PATCH 070/151] Update testrunner.tcl so that it can run the test suites defined in permutation.test. FossilOrigin-Name: 15ce937ef42491c503cb91f5bb3ce4dc5cdb3a927ff2f2b873fb6bf96808f3aa --- manifest | 15 ++-- manifest.uuid | 2 +- test/testrunner.tcl | 206 +++++++++++++++++++++++++++++++++++++------- test/testset.tcl | 121 -------------------------- 4 files changed, 182 insertions(+), 162 deletions(-) delete mode 100644 test/testset.tcl diff --git a/manifest b/manifest index a3df518eea..6074bc96bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sapplying\sthe\somit-ORDER-BY\soptimization,\sdefer\sdeleting\sthe\sAST\sof\nthe\sdeleted\sORDER\sBY\sclause\suntil\safter\scode\sgeneration\sends. -D 2022-07-14T01:48:27.069 +C Update\stestrunner.tcl\sso\sthat\sit\scan\srun\sthe\stest\ssuites\sdefined\sin\spermutation.test. +D 2022-07-14T18:09:56.687 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1507,8 +1507,7 @@ F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 -F test/testrunner.tcl da340a87fd8b5c94e0d2196b086b96a90de0ce16bfdb71c1254d543bcee1614a -F test/testset.tcl 27a6bbbc93cbbcf442c57e3c023e6b5d7304dc415e09eb0e9ac61edd0e6c1fbe +F test/testrunner.tcl 711ccfb6309e63877b20910f76e8b5c8c9e45176cce0efd6f8a40b4e4d44d0e9 F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1981,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14918f28221a3124b78a490fbb483279551ccc5a0032ea854ff0ac365684cc60 -R d2c6af8b43cf81f08214974216426964 -U drh -Z 663440ae7ea06bca4d8f20ca5b3783d1 +P f22f95b838873f1d2a320afe3d0f4e4847948fcd343097b93a9f684a6f66d6ba +R f0a96981c3415711d35db8d014ac10fa +U dan +Z 26ecdc9b9c72535e31216f99411f29ef # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4924411865..57333817f3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f22f95b838873f1d2a320afe3d0f4e4847948fcd343097b93a9f684a6f66d6ba \ No newline at end of file +15ce937ef42491c503cb91f5bb3ce4dc5cdb3a927ff2f2b873fb6bf96808f3aa \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 568c599136..51cb9bfa25 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1,21 +1,47 @@ - #------------------------------------------------------------------------- # Usage: # proc usage {} { set a0 testrunner.tcl - puts stderr "Usage: $a0 ?SWITCHES? ?all|veryquick? ?PATTERNS?" + set ::argv [list] + uplevel [list source $::testdir/permutations.test] + + puts stderr "Usage: $a0 ?SWITCHES? ?PERMUTATION? ?PATTERNS?" puts stderr "" puts stderr "where SWITCHES are:" puts stderr " --jobs NUMBER-OF-JOBS" puts stderr "" + puts stderr "available PERMUTATION values are:" + set ii 0 + foreach name [lsort [array names ::testspec]] { + if {($ii % 3)==0} { puts -nonewline stderr " " } + puts -nonewline stderr [format "% -22s" $name] + if {($ii % 3)==2} { puts stderr "" } + incr ii + } + puts stderr "" + puts stderr "" puts stderr "Examples:" - puts stderr " $a0 # Run veryquick.test tests" - puts stderr " $a0 all # Run all tests" - puts stderr " $a0 veryquick rtree% # Run all test scripts from veryquick.test that match 'rtree%'" - puts stderr " $a0 alter% fts5% # Run all test scripts that match 'alter%' or 'rtree%'" + puts stderr " 1) Run the veryquick tests:" + puts stderr " $a0" + puts stderr " 2) Run all test scripts in the source tree:" + puts stderr " $a0 full" + puts stderr " 2) Run the 'memsubsys1' permutation:" + puts stderr " $a0 memsubsys1" + puts stderr " 3) Run all permutations usually run by \[make fulltest\]" + puts stderr " $a0 release" + puts stderr " 4) Run all scripts that match the pattern 'select%':" + puts stderr " $a0 select%" + puts stderr " $a0 all select%" + puts stderr " $a0 full select%" + puts stderr " 5) Run all scripts that are part of the veryquick permutation and match the pattern 'select%':" + puts stderr " $a0 veryquick select%" + puts stderr " 6) Run the 'memsubsys1' permutation, but just those scripts that match 'window%':" + puts stderr " $a0 memsubsys1 window%" + puts stderr " 7) Run all the permutations, but only the scripts that match either 'fts5%' or 'rtree%':" + puts stderr " $a0 release fts5% rtree%" exit 1 } @@ -30,13 +56,15 @@ set R(schema) { DROP TABLE IF EXISTS malloc; CREATE TABLE script( - filename TEXT PRIMARY KEY, -- full path to test script + config TEXT, + filename TEXT, -- full path to test script state TEXT CHECK( state IN ('ready', 'running', 'done') ), testfixtureid, -- Id of process that ran script time INTEGER, -- Time in ms nerr INTEGER, -- if 'done', the number of errors ntest INTEGER, -- if 'done', the number of tests - output TEXT -- full output of test script + output TEXT, -- full output of test script + PRIMARY KEY(config, filename) ); CREATE TABLE malloc( @@ -95,7 +123,6 @@ set R(leaker) "" ;# Name of first script to leak memory set R(patternlist) [list] set testdir [file dirname $argv0] -source $testdir/testset.tcl # Parse the command line options. There are two ways to invoke this # script - to create a helper or coordinator process. If there are @@ -140,15 +167,86 @@ if {$R(helper)==0} { usage } } else { - lappend R(patternlist) [string map {% * _ .} $a] + lappend R(patternlist) [string map {% *} $a] } } set argv [list] } +source $testdir/permutations.test -source $testdir/tester.tcl -db close +#------------------------------------------------------------------------- +# Return a list of tests to run. Each element of the list is itself a +# list of two elements - the name of a permuations.test configuration +# followed by the full path to a test script. i.e.: +# +# {CONFIG FILENAME} {CONFIG FILENAME} ... +# +proc testset_patternlist {patternlist} { + + set first [lindex $patternlist 0] + if {$first=="all"} { set first "full" } + + if {$first=="release"} { + + # The following mirrors the set of test suites invoked by "all.test". + # + set clist { + full + no_optimization memsubsys1 memsubsys2 singlethread + multithread onefile utf16 exclusive persistent_journal + persistent_journal_error no_journal no_journal_error + autovacuum_ioerr no_mutex_try fullmutex journaltest + inmemory_journal pcache0 pcache10 pcache50 pcache90 + pcache100 prepare mmap + } + ifcapable rbu { lappend clist rbu } + if {$::tcl_platform(platform)=="unix"} { + ifcapable !default_autovacuum { + lappend clist autovacuum_crash + } + } + set patternlist [lrange $patternlist 1 end] + + } elseif {[info exists ::testspec($first)]} { + set clist $first + set patternlist [lrange $patternlist 1 end] + } elseif { [llength $patternlist]==0 } { + set clist veryquick + } else { + set clist full + } + + set testset [list] + + foreach config $clist { + catch { array unset O } + array set O $::testspec($config) + foreach f $O(-files) { + if {[file pathtype $f]!="absolute"} { + set f [file join $::testdir $f] + } + lappend testset [list $config [file normalize $f]] + } + } + + if {[llength $patternlist]>0} { + foreach t $testset { + set tail [file tail [lindex $t 1]] + foreach p $patternlist { + if {[string match $p $tail]} { + lappend ret $t + break; + } + } + } + } else { + set ret $testset + } + + set ret +} +#-------------------------------------------------------------------------- proc r_write_db {tcl} { @@ -166,37 +264,63 @@ proc r_write_db {tcl} { proc make_new_testset {} { global R - set scripts [testset_patternlist $R(patternlist)] + set tests [testset_patternlist $R(patternlist)] r_write_db { db eval $R(schema) - foreach s $scripts { - db eval { INSERT INTO script(filename, state) VALUES ($s, 'ready') } + foreach t $tests { + foreach {c s} $t {} + db eval { + INSERT INTO script(config, filename, state) VALUES ($c, $s, 'ready') + } } } } +# Find the next job in the database and mark it as 'running'. Then return +# a list consisting of the +# +# CONFIG FILENAME +# +# pair for the test. +# proc get_next_test {} { global R set myid $R(helper_id) r_write_db { - set f [db one { - SELECT filename FROM script WHERE state='ready' ORDER BY 1 LIMIT 1 - }] + set f "" + set c "" + db eval { + SELECT config, filename FROM script WHERE state='ready' + ORDER BY config!='full', config, filename LIMIT 1 + } { + set c $config + set f $filename + } if {$f!=""} { db eval { - UPDATE script SET state='running', testfixtureid=$myid WHERE filename=$f + UPDATE script SET state='running', testfixtureid=$myid + WHERE (config, filename) = ($c, $f) } } } - return $f + if {$f==""} { return "" } + list $c $f } -proc r_set_test_result {filename ms nerr ntest output} { +proc r_testname {config filename} { + set name [file tail $filename] + if {$config!="" && $config!="full" && $config!="veryquick"} { + set name "$config-$name" + } + return $name +} + +proc r_set_test_result {config filename ms nerr ntest output} { global R - set f [file tail $filename] + set f [r_testname $config $filename] if {$nerr==0} { set msg "$f... Ok" } else { @@ -211,14 +335,14 @@ proc r_set_test_result {filename ms nerr ntest output} { set nMalloc [lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1] set nByte [sqlite3_memory_used] if {($nByte>0 || $nMalloc>0) && $R(leaker)==""} { - set R(leaker) $filename + set R(leaker) $f } r_write_db { db eval { UPDATE script SET state='done', output=$output, nerr=$nerr, ntest=$ntest, time=$ms - WHERE filename=$filename; + WHERE (config, filename)=($config, $filename); INSERT INTO msg(msg) VALUES ($msg); } @@ -408,7 +532,7 @@ proc r_final_report {} { # Create the text log file. This is just the concatenation of the # 'output' column of the database for every script that was run. set fd [open $R(logname) w] - db eval {SELECT output FROM script ORDER BY filename} { + db eval {SELECT output FROM script ORDER BY config!='full',config,filename} { puts $fd $output } close $fd @@ -421,8 +545,8 @@ proc r_final_report {} { db eval { SELECT sum(nerr) AS nerr, sum(ntest) AS ntest FROM script } { } puts "$nerr errors from $ntest tests." if {$nerr>0} { - db eval { SELECT filename FROM script WHERE nerr>0 } { - lappend errlist [file tail $filename] + db eval { SELECT config, filename FROM script WHERE nerr>0 } { + lappend errlist [r_testname $config $filename] } puts "Errors in: $errlist" set errcode 1 @@ -431,8 +555,8 @@ proc r_final_report {} { # Check if any scripts were not run or did not finish. Print out a # line identifying them if there are any. set errlist [list] - db eval { SELECT filename FROM script WHERE state!='done' } { - lappend errlist [file tail $filename] + db eval { SELECT config, filename FROM script WHERE state!='done' } { + lappend errlist [r_testname $config $filename] } if {$errlist!=[list]} { puts "Tests DID NOT FINISH (crashed?): $errlist" @@ -498,13 +622,31 @@ proc r_helper_readable {id chan} { } if {$R(nHelperRunning)==0} { - while { ""!=[set f [get_next_test]] } { + while { ""!=[set t [get_next_test]] } { set R(output) "" set TC(count) 0 set TC(errors) 0 - set ms [slave_test_file $f] - r_set_test_result $f $ms $TC(errors) $TC(count) $R(output) + foreach {config filename} $t {} + + array set O $::testspec($config) + set ::G(perm:name) $config + set ::G(perm:prefix) $O(-prefix) + set ::G(isquick) 1 + set ::G(perm:dbconfig) $O(-dbconfig) + set ::G(perm:presql) $O(-presql) + + eval $O(-initialize) + set ms [slave_test_file $filename] + eval $O(-shutdown) + + unset -nocomplain ::G(perm:sqlite3_args) + unset ::G(perm:name) + unset ::G(perm:prefix) + unset ::G(perm:dbconfig) + unset ::G(perm:presql) + + r_set_test_result $config $filename $ms $TC(errors) $TC(count) $R(output) if {$R(helper)==0} { foreach msg [r_get_messages] { puts $msg } diff --git a/test/testset.tcl b/test/testset.tcl deleted file mode 100644 index 8f761d686a..0000000000 --- a/test/testset.tcl +++ /dev/null @@ -1,121 +0,0 @@ - - -# Commands in this file: -# -# testset_all -# Return a list of all test scripts designed to be run individually. -# -# testset_veryquick -# The subset of [testset_all] meant to run as veryquick.test. -# - -set D(testdir) [file dir [file normalize [info script]]] - -proc testset_all {} { - global D - set ret [list] - - # The following tests are driver scripts that themselves run lots of other - # test scripts. They should be ignored here. - set drivers { - all.test async.test quick.test veryquick.test - memleak.test permutations.test soak.test fts3.test - mallocAll.test rtree.test full.test extraquick.test - session.test rbu.test - } - - set srcdir [file dirname $D(testdir)] - set ret [glob -nocomplain \ - $srcdir/test/*.test \ - $srcdir/ext/rtree/*.test \ - $srcdir/ext/fts5/test/*.test \ - $srcdir/ext/expert/*.test \ - $srcdir/ext/session/*.test \ - ] - set ret [ts_filter $ret $drivers] - return $ret -} - -proc testset_veryquick {} { - set ret [testset_all] - - set ret [ts_filter $ret { - async2.test async3.test backup_ioerr.test corrupt.test - corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test - crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test - fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test - misc7.test mutex2.test notify2.test onefile.test pagerfault2.test - savepoint4.test savepoint6.test select9.test - speed1.test speed1p.test speed2.test speed3.test speed4.test - speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test - thread003.test thread004.test thread005.test trans2.test vacuum3.test - incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test - vtab_err.test walslow.test walcrash.test walcrash3.test - walthread.test rtree3.test indexfault.test securedel2.test - sort3.test sort4.test fts4growth.test fts4growth2.test - bigsort.test walprotocol.test mmap4.test fuzzer2.test - walcrash2.test e_fkey.test backup.test - - fts4merge.test fts4merge2.test fts4merge4.test fts4check.test - fts4merge5.test - fts3cov.test fts3snippet.test fts3corrupt2.test fts3an.test - fts3defer.test fts4langid.test fts3sort.test fts5unicode.test - - rtree4.test - sessionbig.test - }] - - set ret [ts_filter $ret { - *malloc* *ioerr* *fault* *bigfile* *_err* *fts5corrupt* *fts5big* *fts5aj* - }] - - return $ret -} - -proc ts_filter {input exlist} { - foreach f $input { set a($f) 1 } - foreach e $exlist { array unset a */$e } - array names a -} - -proc testset_patternlist {patternlist} { - set nPat [llength $patternlist] - - if {$nPat==0} { - set scripts [testset_veryquick] - } else { - set ii 0 - set p0 [lindex $patternlist 0] - - if {$p0=="veryquick"} { - set scripts [testset_veryquick] - incr ii - } elseif {$p0=="all"} { - set scripts [testset_all] - incr ii - } else { - set scripts [testset_all] - } - - if {$nPat>$ii} { - array set S [list] - foreach f $scripts { set a([file tail $f]) $f } - - foreach p [lrange $patternlist $ii end] { - set nList [llength [array names a $p]] - if {$nList==0} { - puts stderr "Argument $p matches no scripts (typo?)" - exit 1 - } - foreach n [array names a $p] { set S($a($n)) 1 } - } - - set scripts [lsort [array names S]] - } - - } - - set scripts -} - - From de353fb5c22f0f0a2eae1d6b418ba8f3614a4c0c Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Jul 2022 21:17:22 +0000 Subject: [PATCH 071/151] Ensure all testrunner.tcl processes use the same pending-byte value when accessing testrunner.db. Otherwise locking doesn't work and the db is corrupted. FossilOrigin-Name: b65225653e4e5c20cc43f4dfdde6c110f6d14b87bff02a65fd61beabe50ec8db --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6074bc96bc..f620d76794 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stestrunner.tcl\sso\sthat\sit\scan\srun\sthe\stest\ssuites\sdefined\sin\spermutation.test. -D 2022-07-14T18:09:56.687 +C Ensure\sall\stestrunner.tcl\sprocesses\suse\sthe\ssame\spending-byte\svalue\swhen\naccessing\stestrunner.db.\sOtherwise\slocking\sdoesn't\swork\sand\sthe\sdb\sis\scorrupted. +D 2022-07-14T21:17:22.174 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1507,7 +1507,7 @@ F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 -F test/testrunner.tcl 711ccfb6309e63877b20910f76e8b5c8c9e45176cce0efd6f8a40b4e4d44d0e9 +F test/testrunner.tcl 6aabdfcbfc489cc666720048606e5f9e62b8e17a3cfaf36a81fc587d9e7c39cb F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1980,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f22f95b838873f1d2a320afe3d0f4e4847948fcd343097b93a9f684a6f66d6ba -R f0a96981c3415711d35db8d014ac10fa +P 15ce937ef42491c503cb91f5bb3ce4dc5cdb3a927ff2f2b873fb6bf96808f3aa +R 4d50bb0a2ba6c649edd8a048f838a623 U dan -Z 26ecdc9b9c72535e31216f99411f29ef +Z 53d849ffc9222974909e46390d4cb28e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 57333817f3..491f3570e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15ce937ef42491c503cb91f5bb3ce4dc5cdb3a927ff2f2b873fb6bf96808f3aa \ No newline at end of file +b65225653e4e5c20cc43f4dfdde6c110f6d14b87bff02a65fd61beabe50ec8db \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 51cb9bfa25..c471473d90 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -251,6 +251,8 @@ proc testset_patternlist {patternlist} { proc r_write_db {tcl} { global R + + sqlite3_test_control_pending_byte 0x010000 sqlite3 db $R(dbname) db timeout $R(timeout) db eval { BEGIN EXCLUSIVE } @@ -353,6 +355,8 @@ set R(iNextMsg) 1 proc r_get_messages {{db ""}} { global R + sqlite3_test_control_pending_byte 0x010000 + if {$db==""} { sqlite3 rgmhandle $R(dbname) set dbhandle rgmhandle @@ -524,7 +528,9 @@ proc puts_into_caller {args} { proc r_final_report {} { global R + sqlite3_test_control_pending_byte 0x010000 sqlite3 db $R(dbname) + db timeout $R(timeout) set errcode 0 @@ -613,7 +619,7 @@ if {$R(helper)==0 && $R(nJob)>1} { proc r_helper_readable {id chan} { set data [gets $chan] - if {$data!=""} { puts "helper $id:[gets $chan]" } + if {$data!=""} { puts "helper $id:$data" } if {[eof $chan]} { puts "helper $id is finished" incr ::R(nHelperRunning) -1 From b1c034b2f1858fc308396c34b8f760cdbe7a3966 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Jul 2022 11:34:23 +0000 Subject: [PATCH 072/151] Update some faulty assert() statements in fts3. FossilOrigin-Name: b072851be1a5e49441469dbfa4580e132ebb1d8bc2de98fe1aa67bb9c7cf13a3 --- ext/fts3/fts3.c | 3 - manifest | 14 +-- manifest.uuid | 2 +- test/fts3corrupt4.test | 219 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 227 insertions(+), 11 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 7a3c735e80..f601543ef8 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -4309,8 +4309,6 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ int nPoslist = 0; /* Number of bytes in aPoslist */ int iPrev = -1; /* Token number of previous deferred token */ - assert( pPhrase->doclist.bFreeList==0 ); - for(iToken=0; iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; Fts3DeferredToken *pDeferred = pToken->pDeferred; @@ -5573,7 +5571,6 @@ static int fts3EvalTestExpr( pExpr->iDocid==pCsr->iPrevId && pExpr->pPhrase->doclist.pList ))){ Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); if( pExpr->bDeferred ){ fts3EvalInvalidatePoslist(pPhrase); } diff --git a/manifest b/manifest index f620d76794..2dcfc2f17e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sall\stestrunner.tcl\sprocesses\suse\sthe\ssame\spending-byte\svalue\swhen\naccessing\stestrunner.db.\sOtherwise\slocking\sdoesn't\swork\sand\sthe\sdb\sis\scorrupted. -D 2022-07-14T21:17:22.174 +C Update\ssome\sfaulty\sassert()\sstatements\sin\sfts3. +D 2022-07-15T11:34:23.621 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -102,7 +102,7 @@ F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 5577b9e88b492e6dab0592c2367d82ae3a634109a0903d0c9f9db69121da1d85 +F ext/fts3/fts3.c 9ba24e484da5146f033ee78ecf6bc740fc4f62c6c42143f9f2c99be27baca4c8 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h dafdc371f9fbab175744b06cfe019d5f040cdfdbd11fea752f5dc28d45b04c05 F ext/fts3/fts3_aux.c f0dc9bd98582615b7750218899bd0c729879b6bbf94d1be57ca1833ff49afc6f @@ -1008,7 +1008,7 @@ F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f F test/fts3corrupt.test 6732477c5ace050c5758a40a8b5706c8c0cccd416b9c558e0e15224805a40e57 F test/fts3corrupt2.test e318f0676e5e78d5a4b702637e2bb25265954c08a1b1e4aaf93c7880bb0c67d0 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f -F test/fts3corrupt4.test 799ff994b964fed7201be6b6b62c7ff2ef7bb3da6c02b9eaf0d96a5a4d9b6ca3 +F test/fts3corrupt4.test e3f7915a156d75bec2fb282630165197d8e682498c5177d43dfd453f5a4ede80 F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5 F test/fts3corrupt6.test f417c910254f32c0bc9ead7affa991a1d5aec35b3b32a183ffb05eea78289525 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf @@ -1980,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 15ce937ef42491c503cb91f5bb3ce4dc5cdb3a927ff2f2b873fb6bf96808f3aa -R 4d50bb0a2ba6c649edd8a048f838a623 +P b65225653e4e5c20cc43f4dfdde6c110f6d14b87bff02a65fd61beabe50ec8db +R 85d2bb5585b425f8a8c86dc6c1bf3ff5 U dan -Z 53d849ffc9222974909e46390d4cb28e +Z fe70f0d7f39c4194455d1661c206e596 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 491f3570e1..e2ab433e0f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b65225653e4e5c20cc43f4dfdde6c110f6d14b87bff02a65fd61beabe50ec8db \ No newline at end of file +b072851be1a5e49441469dbfa4580e132ebb1d8bc2de98fe1aa67bb9c7cf13a3 \ No newline at end of file diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test index f0b83a3901..1e9dae2a4c 100644 --- a/test/fts3corrupt4.test +++ b/test/fts3corrupt4.test @@ -7033,6 +7033,8 @@ set sqlite_fts3_enable_parentheses $saved #------------------------------------------------------------------------- # +set saved $sqlite_fts3_enable_parentheses +set sqlite_fts3_enable_parentheses 1 reset_db do_test 52.0 { sqlite3 db {} @@ -7228,5 +7230,222 @@ do_catchsql_test 52.1 { SELECT * FROM t1, t2; } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +# +reset_db +do_test 53.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 8192 pagesize 1024 filename crash-7bc.txt.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 04 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 02 f3 00 07 01 51 00 03 c8 03 63 ..........Q....c +| 112: 02 fb 02 0a 02 c0 01 a8 01 51 00 00 00 00 00 00 .........Q...... +| 336: 00 55 07 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 .U........tablet +| 352: 31 5f 73 74 61 74 74 31 5f 73 74 61 74 07 43 52 1_statt1_stat.CR +| 368: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 73 EATE TABLE 't1_s +| 384: 74 61 74 27 28 69 64 20 49 4e 54 45 47 45 52 20 tat'(id INTEGER +| 400: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 61 6c PRIMARY KEY, val +| 416: 75 65 20 42 4c 4f 42 29 60 06 07 17 21 21 01 81 ue BLOB)`...!!.. +| 432: 0b 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 .tablet1_docsize +| 448: 74 31 5f 64 6f 63 73 69 7a 65 06 43 52 45 41 54 t1_docsize.CREAT +| 464: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs +| 480: 69 7a 65 27 28 64 6f 63 69 64 20 49 4e 54 45 47 ize'(docid INTEG +| 496: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 512: 73 69 7a 65 20 42 4c 4f 42 29 81 33 04 07 17 1f size BLOB).3.... +| 528: 1f 01 82 35 74 61 62 6c 65 74 31 5f 73 65 67 64 ...5tablet1_segd +| 544: 69 72 74 31 5f 73 65 67 64 69 72 04 43 52 45 41 irt1_segdir.CREA +| 560: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 73 65 67 TE TABLE 't1_seg +| 576: 64 69 72 27 28 6c 65 76 65 6c 20 49 4e 54 45 47 dir'(level INTEG +| 592: 45 52 2c 69 64 78 20 49 4e 54 45 47 45 52 2c 73 ER,idx INTEGER,s +| 608: 74 61 72 74 5f 62 6c 6f 63 6b 20 49 4e 54 45 47 tart_block INTEG +| 624: 45 52 2c 6c 65 61 76 65 73 5f 65 6e 64 5f 62 6c ER,leaves_end_bl +| 640: 6f 63 6b 20 49 4e 54 45 47 45 52 2c 65 6e 64 5f ock INTEGER,end_ +| 656: 62 6c 6f 63 6b 20 49 4e 54 45 47 45 52 2c 72 6f block INTEGER,ro +| 672: 6f 74 20 42 4c 4f 42 2c 50 52 49 4d 41 52 59 20 ot BLOB,PRIMARY +| 688: 4b 45 59 28 6c 65 76 65 6c 2c 20 69 64 78 29 29 KEY(level, idx)) +| 704: 31 05 06 17 45 1f 01 00 69 6e 64 65 78 73 71 6c 1...E...indexsql +| 720: 69 74 65 5f 61 75 74 6f 69 6e 64 65 78 5f 74 31 ite_autoindex_t1 +| 736: 5f 73 65 67 64 69 72 5f 31 74 31 5f 73 65 67 64 _segdir_1t1_segd +| 752: 69 72 05 00 00 00 08 00 00 00 00 66 03 07 17 23 ir.........f...# +| 768: 23 01 81 13 74 61 62 6c 65 74 31 5f 73 65 67 6d #...tablet1_segm +| 784: 65 6e 74 73 74 31 5f 73 65 67 6d 65 6e 74 73 03 entst1_segments. +| 800: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 816: 5f 73 65 67 6d 65 6e 74 73 27 28 62 6c 6f 63 6b _segments'(block +| 832: 69 64 20 49 4e 53 45 47 45 52 20 50 52 49 4d 41 id INSEGER PRIMA +| 848: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 864: 4f 42 29 63 02 07 17 21 21 01 81 11 74 61 62 6c OB)c...!!...tabl +| 880: 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f et1_contentt1_co +| 896: 6e 74 65 6e 74 02 43 52 45 41 54 45 20 54 41 42 ntent.CREATE TAB +| 912: 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 LE 't1_content'( +| 928: 64 6f 63 69 64 20 49 4e 54 45 47 45 52 20 50 52 docid INTEGER PR +| 944: 49 4d 41 52 59 20 4b 45 59 2c 20 27 63 30 30 27 IMARY KEY, 'c00' +| 960: 2c 20 27 63 31 62 27 29 36 01 06 17 11 11 08 5b , 'c1b')6......[ +| 976: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE +| 992: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 1008: 20 55 53 49 4e 47 20 66 74 73 34 28 30 2c 62 29 USING fts4(0,b) +| page 2 offset 1024 +| 0: 0d 00 00 00 03 00 0f 00 00 23 00 16 00 0f 00 05 .........#...... +| 16: 03 04 00 08 0f 61 0b 02 04 00 08 1b 41 54 45 20 .....a......ATE +| 32: 32 3a 50 87 5a 01 05 00 08 8f 37 66 30 30 30 30 2:P.Z.....7f0000 +| 48: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 64: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 80: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 96: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 112: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 128: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 144: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 160: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 176: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 192: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 208: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 224: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 240: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 256: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 272: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 288: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 304: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 320: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 336: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 352: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 368: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 384: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 400: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 416: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 432: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 448: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 464: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 480: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 496: 30 40 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0@00000000000000 +| 512: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 528: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 544: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 560: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 576: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 592: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 608: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 624: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 640: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 656: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 672: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 688: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 704: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 720: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 736: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 752: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 768: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 784: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 800: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 816: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 832: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 848: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 864: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 880: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 896: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 912: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 928: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 944: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 960: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 976: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 992: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 1008: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| page 3 offset 2048 +| 0: 0d 00 00 00 02 03 86 00 03 f4 03 86 00 00 00 00 ................ +| 896: 00 00 00 00 00 00 87 62 02 04 00 8f 48 00 d5 07 .......b....H... +| 912: 66 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 f000000000000000 +| 928: 30 30 30 30 3a 30 30 30 30 30 30 30 30 30 30 30 0000:00000000000 +| 944: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 960: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 976: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 992: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 1008: 00 00 00 08 0a 01 03 00 1a 00 01 30 03 01 02 00 ...........0.... +| page 4 offset 3072 +| 0: 0d 00 00 00 03 03 9e 00 03 ed 03 bc 03 9e 00 00 ................ +| 912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1c 03 ................ +| 928: 07 08 01 08 08 15 2c 02 30 20 31 36 00 01 30 03 ......,.0 16..0. +| 944: 03 02 00 00 01 61 05 03 01 01 02 00 2f 02 07 08 .....a....../... +| 960: 09 08 08 15 54 30 20 33 36 00 01 30 03 02 02 00 ....T0 36..0.... +| 976: 00 01 32 05 02 01 01 03 00 00 03 61 74 65 05 02 ..2........ate.. +| 992: 01 01 02 00 00 01 70 05 02 01 01 04 00 11 01 07 ......p......... +| 1008: 08 08 09 01 17 14 02 32 20 39 39 37 01 01 01 66 .......2 997...f +| page 5 offset 4096 +| 0: 0a 00 00 00 03 03 ee 00 03 fb 03 f5 03 ee 00 00 ................ +| 992: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 04 ................ +| 1008: 08 01 01 02 03 05 04 08 09 01 02 04 04 08 08 09 ................ +| page 6 offset 5120 +| 0: 0d 00 00 00 03 03 eb 00 00 00 00 00 00 00 00 00 ................ +| 992: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 1008: 01 01 05 02 03 00 10 01 03 05 01 03 00 10 01 01 ................ +| page 7 offset 6144 +| 0: 0d 00 00 00 01 03 f6 00 03 f6 00 00 00 00 00 00 ................ +| 1008: 00 00 00 00 00 00 08 00 03 00 16 03 08 c5 e0 07 ................ +| page 8 offset 7168 +| 0: 00 00 00 00 30 30 30 30 30 30 30 30 30 30 30 30 ....000000000000 +| 16: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 32: 30 30 30 30 30 30 30 30 30 30 30 30 30 bc 30 30 0000000000000.00 +| 48: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 64: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 80: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 96: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 112: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 128: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 144: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 160: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 176: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 192: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 c0 30 00000000000000.0 +| 208: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 224: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 240: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 256: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 272: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 288: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 304: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 320: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 336: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 352: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 368: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 384: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 400: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 416: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 432: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 448: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 464: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 480: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 496: 30 30 30 30 30 30 30 30 30 30 30 40 30 30 30 30 00000000000@0000 +| 512: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 528: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 544: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 560: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 576: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 592: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 608: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 624: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 640: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 656: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 672: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 688: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 704: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 720: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 736: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 752: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 768: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 784: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 800: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 816: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 832: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 848: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 864: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 880: 30 30 30 30 30 30 30 30 30 05 01 00 00 00 00 00 000000000....... +| end crash-7bc.txt.db +}]} {} + +do_execsql_test 53.1 { + SELECT*FROM t1 WHERE t1 MATCH'ATE"0"OR"2D:P"""ATE"0"OR"2:P"""'; +} {0 {ATE 2:P}} +set sqlite_fts3_enable_parentheses $saved finish_test From b3623e0af762a756074475dd91c88a4abd44a2c7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 15 Jul 2022 12:16:11 +0000 Subject: [PATCH 073/151] Fix the whereKeyStats() routine (part of STAT4 processing only) so that it is able to cope with row-value comparisons against the primary key index of a WITHOUT ROWID table. [forum:/forumpost/3607259d3c|Forum post 3607259d3c]. FossilOrigin-Name: 0620e419a927a3da6ebe921aaa3471686f0fdc2e485f4c2d5c88f32092228724 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 4 ++-- test/rowvalue.test | 31 +++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 2dcfc2f17e..0889a02db4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\ssome\sfaulty\sassert()\sstatements\sin\sfts3. -D 2022-07-15T11:34:23.621 +C Fix\sthe\swhereKeyStats()\sroutine\s(part\sof\sSTAT4\sprocessing\sonly)\sso\sthat\sit\nis\sable\sto\scope\swith\srow-value\scomparisons\sagainst\sthe\sprimary\skey\sindex\nof\sa\sWITHOUT\sROWID\stable.\n[forum:/forumpost/3607259d3c|Forum\spost\s3607259d3c]. +D 2022-07-15T12:16:11.966 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -657,7 +657,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 5af9a9773a3bb84acf4eac8b92b7951b780ece1dca914223f78effc4effeca2d +F src/where.c c7e96c036d2bf0755a2bedf796951ff8bf0dd062b07c4ccc724e1432c3dcf5db F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 F src/wherecode.c 0b09abfcb88c61c6a6984a3e065786631ff35495e9bdf865e6b74ab0a1299c5b F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 @@ -1358,7 +1358,7 @@ F test/round1.test 768018b04522ca420b1aba8a24bd76091d269f3bce3902af3ec6ebcee41ab F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test e29025be95baf6b32f0d5edef59a7633028325896a98f1caa8019559ca910350 -F test/rowvalue.test ff1ffa31cebe12feb6f989e09263f3b1e8c560db94b40fe006126a8435fd6832 +F test/rowvalue.test baf4fa3ec1a8c1c920c3faa5fd25959cb454bbd99ac8960397c34549d9fc4abe F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 1347e25ca11c547c5a6ff0cc5626f95aa9740e9275bfaec096029f57cb2130ce F test/rowvalue4.test 441e7e366ac6d939a3a95a574031c56ec2a854077a91d66eee5ff1d86cb5be58 @@ -1980,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b65225653e4e5c20cc43f4dfdde6c110f6d14b87bff02a65fd61beabe50ec8db -R 85d2bb5585b425f8a8c86dc6c1bf3ff5 -U dan -Z fe70f0d7f39c4194455d1661c206e596 +P b072851be1a5e49441469dbfa4580e132ebb1d8bc2de98fe1aa67bb9c7cf13a3 +R 09fb2c54faf3b89c4b078e3ba6b760f0 +U drh +Z ea98894b53d9b6ede773b2a171a6b669 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e2ab433e0f..d6fa150f9d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b072851be1a5e49441469dbfa4580e132ebb1d8bc2de98fe1aa67bb9c7cf13a3 \ No newline at end of file +0620e419a927a3da6ebe921aaa3471686f0fdc2e485f4c2d5c88f32092228724 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 405811a566..3dd5058fc9 100644 --- a/src/where.c +++ b/src/where.c @@ -1433,7 +1433,7 @@ static int whereKeyStats( #endif assert( pRec!=0 ); assert( pIdx->nSample>0 ); - assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol ); + assert( pRec->nField>0 ); /* Do a binary search to find the first sample greater than or equal ** to pRec. If pRec contains a single field, the set of samples to search @@ -1479,7 +1479,7 @@ static int whereKeyStats( ** it is extended to two fields. The duplicates that this creates do not ** cause any problems. */ - nField = pRec->nField; + nField = MIN(pRec->nField, pIdx->nSample); iCol = 0; iSample = pIdx->nSample * nField; do{ diff --git a/test/rowvalue.test b/test/rowvalue.test index 12fee82376..59b44d9386 100644 --- a/test/rowvalue.test +++ b/test/rowvalue.test @@ -751,4 +751,35 @@ do_execsql_test 32.1 { WHERE a=1234 OR a<=567; } {500 502} +# 2022-07-15 +# https://sqlite.org/forum/forumpost/3607259d3c +# +reset_db +do_execsql_test 33.1 { + CREATE TABLE t1(a INT, b INT PRIMARY KEY) WITHOUT ROWID; + INSERT INTO t1(a, b) VALUES (0, 1),(15,-7),(3,100); + ANALYZE; +} {} +do_execsql_test 33.2 { + SELECT * FROM t1 WHERE (b,a) BETWEEN (0,5) AND (99,-2); +} {0 1} +do_execsql_test 33.3 { + SELECT * FROM t1 WHERE (b,a) BETWEEN (-8,5) AND (0,-2); +} {15 -7} +do_execsql_test 33.3 { + SELECT * FROM t1 WHERE (b,a) BETWEEN (3,5) AND (100,4); +} {3 100} +do_execsql_test 33.3 { + SELECT * FROM t1 WHERE (b,a) BETWEEN (3,5) AND (100,2); +} {} +do_execsql_test 33.3 { + SELECT * FROM t1 WHERE (a,b) BETWEEN (-2,99) AND (1,0); +} {0 1} +do_execsql_test 33.3 { + SELECT * FROM t1 WHERE (a,b) BETWEEN (14,99) AND (16,0); +} {15 -7} +do_execsql_test 33.3 { + SELECT * FROM t1 WHERE (a,b) BETWEEN (2,99) AND (4,0); +} {3 100} + finish_test From 40e3fa30357b549abf32ddadaf654b277c12c912 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Jul 2022 15:08:48 +0000 Subject: [PATCH 074/151] Fix a memory leak in fts3 that could occur when processing a corrupt database. FossilOrigin-Name: d74f6f6d5136995b8bf900eb671e4b15ca81e03cc1ab5b7a1aa43dc4f3617760 --- ext/fts3/fts3.c | 4 + manifest | 16 +-- manifest.uuid | 2 +- test/fts3corrupt4.test | 216 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 229 insertions(+), 9 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index f601543ef8..1a1672512a 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -4308,6 +4308,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ char *aPoslist = 0; /* Position list for deferred tokens */ int nPoslist = 0; /* Number of bytes in aPoslist */ int iPrev = -1; /* Token number of previous deferred token */ + char *aFree = (pPhrase->doclist.bFreeList ? pPhrase->doclist.pList : 0); for(iToken=0; iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; @@ -4321,6 +4322,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ if( pList==0 ){ sqlite3_free(aPoslist); + sqlite3_free(aFree); pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; return SQLITE_OK; @@ -4341,6 +4343,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nPoslist = (int)(aOut - aPoslist); if( nPoslist==0 ){ sqlite3_free(aPoslist); + sqlite3_free(aFree); pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; return SQLITE_OK; @@ -4393,6 +4396,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ } } + if( pPhrase->doclist.pList!=aFree ) sqlite3_free(aFree); return SQLITE_OK; } #endif /* SQLITE_DISABLE_FTS4_DEFERRED */ diff --git a/manifest b/manifest index 0889a02db4..4c513ab28f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\swhereKeyStats()\sroutine\s(part\sof\sSTAT4\sprocessing\sonly)\sso\sthat\sit\nis\sable\sto\scope\swith\srow-value\scomparisons\sagainst\sthe\sprimary\skey\sindex\nof\sa\sWITHOUT\sROWID\stable.\n[forum:/forumpost/3607259d3c|Forum\spost\s3607259d3c]. -D 2022-07-15T12:16:11.966 +C Fix\sa\smemory\sleak\sin\sfts3\sthat\scould\soccur\swhen\sprocessing\sa\scorrupt\sdatabase. +D 2022-07-15T15:08:48.874 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -102,7 +102,7 @@ F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 9ba24e484da5146f033ee78ecf6bc740fc4f62c6c42143f9f2c99be27baca4c8 +F ext/fts3/fts3.c 8cd361bd8612de3e5ec451d33ced8575cbc5af6744e7efc05d54bde4d2df3cd0 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h dafdc371f9fbab175744b06cfe019d5f040cdfdbd11fea752f5dc28d45b04c05 F ext/fts3/fts3_aux.c f0dc9bd98582615b7750218899bd0c729879b6bbf94d1be57ca1833ff49afc6f @@ -1008,7 +1008,7 @@ F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f F test/fts3corrupt.test 6732477c5ace050c5758a40a8b5706c8c0cccd416b9c558e0e15224805a40e57 F test/fts3corrupt2.test e318f0676e5e78d5a4b702637e2bb25265954c08a1b1e4aaf93c7880bb0c67d0 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f -F test/fts3corrupt4.test e3f7915a156d75bec2fb282630165197d8e682498c5177d43dfd453f5a4ede80 +F test/fts3corrupt4.test 589e043d1114ea02c83530e459ef5c2d6adce63094e72826ed3bceb02e795116 F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5 F test/fts3corrupt6.test f417c910254f32c0bc9ead7affa991a1d5aec35b3b32a183ffb05eea78289525 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf @@ -1980,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b072851be1a5e49441469dbfa4580e132ebb1d8bc2de98fe1aa67bb9c7cf13a3 -R 09fb2c54faf3b89c4b078e3ba6b760f0 -U drh -Z ea98894b53d9b6ede773b2a171a6b669 +P 0620e419a927a3da6ebe921aaa3471686f0fdc2e485f4c2d5c88f32092228724 +R b9aac4d9609cc8841634db9ea3e3fcba +U dan +Z bb096f0e437aaa23189ce1ee6aae46bf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d6fa150f9d..78ed53ebc3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0620e419a927a3da6ebe921aaa3471686f0fdc2e485f4c2d5c88f32092228724 \ No newline at end of file +d74f6f6d5136995b8bf900eb671e4b15ca81e03cc1ab5b7a1aa43dc4f3617760 \ No newline at end of file diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test index 1e9dae2a4c..f8e89b3a75 100644 --- a/test/fts3corrupt4.test +++ b/test/fts3corrupt4.test @@ -7448,4 +7448,220 @@ do_execsql_test 53.1 { } {0 {ATE 2:P}} set sqlite_fts3_enable_parentheses $saved +#------------------------------------------------------------------------- +# +reset_db +do_test 54.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 8192 pagesize 1024 filename crash-365.txt.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 04 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 96: 00 00 00 00 0d 02 f3 00 07 01 51 00 03 c8 03 63 ..........Q....c +| 112: 02 fb 02 0a 02 c0 01 a8 01 51 00 00 00 00 00 00 .........Q...... +| 336: 00 55 07 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 .U........tablet +| 352: 31 5f 73 74 61 74 74 31 5f 73 74 61 74 07 43 52 1_statt1_stat.CR +| 368: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 73 EATE TABLE 't1_s +| 384: 74 61 74 27 28 69 64 20 49 4e 54 45 47 45 52 20 tat'(id INTEGER +| 400: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 61 6c PRIMARY KEY, val +| 416: 75 65 20 42 4c 4f 42 29 60 06 07 17 21 21 01 81 ue BLOB)`...!!.. +| 432: 0b 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 .tablet1_docsize +| 448: 74 31 5f 64 6f 63 73 69 7a 65 06 43 52 45 41 54 t1_docsize.CREAT +| 464: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs +| 480: 69 7a 65 27 28 64 6f 63 69 64 20 49 4e 54 45 47 ize'(docid INTEG +| 496: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 512: 73 69 7a 65 20 42 4c 4f 42 29 81 33 04 07 17 1f size BLOB).3.... +| 528: 1f 01 82 35 74 61 62 6c 65 74 31 5f 73 65 67 64 ...5tablet1_segd +| 544: 69 72 74 31 5f 73 65 67 64 69 72 04 43 52 45 41 irt1_segdir.CREA +| 560: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 73 65 67 TE TABLE 't1_seg +| 576: 64 69 72 27 28 6c 65 76 65 6c 20 49 4e 54 45 47 dir'(level INTEG +| 592: 45 52 2c 69 64 78 20 49 4e 54 45 47 45 52 2c 73 ER,idx INTEGER,s +| 608: 74 61 72 74 5f 62 6c 6f 63 6b 20 49 4e 54 45 47 tart_block INTEG +| 624: 45 52 2c 6c 65 61 76 65 73 5f 65 6e 64 5f 62 6c ER,leaves_end_bl +| 640: 6f 63 6b 20 49 4e 54 45 47 45 52 2c 65 6e 64 5f ock INTEGER,end_ +| 656: 62 6c 6f 63 6b 20 49 4e 54 45 47 45 52 2c 72 6f block INTEGER,ro +| 672: 6f 74 20 42 4c 4f 42 2c 50 52 49 4d 41 52 59 20 ot BLOB,PRIMARY +| 688: 4b 45 59 28 6c 65 76 65 6c 2c 20 69 64 78 29 29 KEY(level, idx)) +| 704: 31 05 06 17 45 1f 01 00 69 6e 64 65 78 73 71 6c 1...E...indexsql +| 720: 69 74 65 5f 61 75 74 6f 69 6e 64 65 78 5f 74 31 ite_autoindex_t1 +| 736: 5f 73 65 67 64 69 72 5f 31 74 31 5f 73 65 67 64 _segdir_1t1_segd +| 752: 69 72 05 00 00 00 08 00 00 00 00 66 03 07 17 23 ir.........f...# +| 768: 23 01 81 13 74 61 62 6c 65 74 31 5f 73 65 67 6d #...tablet1_segm +| 784: 65 6e 74 73 74 31 5f 73 65 67 6d 65 6e 74 73 03 entst1_segments. +| 800: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 816: 5f 73 65 67 6d 65 6e 74 73 27 28 62 6c 6f 63 6b _segments'(block +| 832: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 848: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 864: 4f 42 29 63 02 07 17 21 21 01 81 11 74 61 62 6c OB)c...!!...tabl +| 880: 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f et1_contentt1_co +| 896: 6e 74 65 6e 74 02 43 52 45 41 54 45 20 54 41 42 ntent.CREATE TAB +| 912: 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 LE 't1_content'( +| 928: 64 6f 63 69 64 20 49 4e 54 45 47 45 52 20 50 52 docid INTEGER PR +| 944: 49 4d 41 52 59 20 4b 45 59 2c 20 27 63 30 30 27 IMARY KEY, 'c00' +| 960: 2c 20 27 63 31 62 27 29 36 01 06 17 11 11 08 5b , 'c1b')6......[ +| 976: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE +| 992: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 1008: 20 55 53 49 4e 47 20 66 74 73 34 28 30 2c 62 29 USING fts4(0,b) +| page 2 offset 1024 +| 0: 0d 00 00 00 03 00 0f 00 00 23 00 16 00 0f 00 05 .........#...... +| 16: 03 04 00 08 0f 61 0b 02 04 00 08 1b 41 54 45 20 .....a......ATE +| 32: 32 3a 50 87 5a 01 05 00 08 8f 37 66 30 30 30 30 2:P.Z.....7f0000 +| 48: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 64: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 80: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 96: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 112: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 128: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 144: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 160: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 176: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 192: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 208: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 224: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 240: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 256: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 272: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 288: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 304: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 320: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 336: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 352: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 368: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 384: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 400: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 416: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 432: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 448: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 464: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 480: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 496: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 512: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 528: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 544: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 560: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 576: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 592: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 608: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 624: 30 30 30 30 30 30 30 30 30 30 1b 30 30 30 30 30 0000000000.00000 +| 640: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 656: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 672: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 688: 30 30 30 30 30 30 30 30 2f 30 30 30 30 30 30 30 00000000/0000000 +| 704: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 720: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 736: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 752: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 768: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 784: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 800: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 816: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 832: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 848: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 864: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 880: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 896: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 912: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 928: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 944: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 960: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 976: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 992: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 1008: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| page 3 offset 2048 +| 0: 0d 00 00 00 02 03 86 00 03 f4 03 86 00 00 00 00 ................ +| 896: 00 00 00 00 00 00 87 62 02 04 00 8f 48 00 d5 07 .......b....H... +| 912: 66 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 f000000000000000 +| 928: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 944: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 960: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 976: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 992: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 1008: 00 00 00 08 0a 01 03 00 1a 00 01 30 03 01 02 00 ...........0.... +| page 4 offset 3072 +| 0: 0d 00 00 00 03 03 9e 00 03 ed 03 bc 03 9e 00 01 ................ +| 912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1c 03 ................ +| 928: 07 08 01 08 08 15 2c 02 30 20 31 36 00 01 30 03 ......,.0 16..0. +| 944: 03 02 00 00 01 61 05 03 01 01 02 00 2f 02 07 08 .....a....../... +| 960: 09 08 08 15 54 30 20 33 36 00 01 30 03 02 02 00 ....T0 36..0.... +| 976: 00 01 32 05 02 01 01 03 00 00 03 61 74 65 05 02 ..2........ate.. +| 992: 01 01 02 00 00 01 70 05 02 01 01 04 00 11 01 07 ......p......... +| 1008: 08 08 09 01 17 14 02 32 20 39 39 37 01 01 01 66 .......2 997...f +| page 5 offset 4096 +| 0: 0a 00 00 00 03 03 ee 00 03 fb 03 f5 03 ee 00 00 ................ +| 992: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 04 ................ +| 1008: 08 01 01 02 03 05 04 08 09 01 02 04 04 08 08 09 ................ +| page 6 offset 5120 +| 0: 0d 00 00 00 03 03 eb 00 03 f9 03 f2 00 00 00 00 ................ +| 992: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 1008: 01 01 05 02 03 00 10 01 03 05 01 03 00 10 01 01 ................ +| page 7 offset 6144 +| 0: 0d 00 00 00 01 03 f6 00 03 f6 00 00 00 00 00 00 ................ +| 1008: 00 00 00 00 00 00 08 00 03 00 16 03 03 05 e0 07 ................ +| page 8 offset 7168 +| 0: 00 00 00 00 30 30 30 30 30 30 30 30 30 30 30 30 ....000000000000 +| 16: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 32: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 48: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 64: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 80: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 96: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 112: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 128: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 144: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 160: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 176: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 192: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 208: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 224: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 240: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 256: 30 30 30 30 30 2f 30 30 30 30 30 30 30 30 30 30 00000/0000000000 +| 272: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 288: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 304: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 320: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 336: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 352: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 368: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 384: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 400: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 416: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 432: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 448: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 464: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 480: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 496: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 512: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 528: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 544: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 560: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 576: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 592: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 608: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 624: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 640: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 656: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 672: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 688: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 704: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 720: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 736: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 752: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 768: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 784: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 800: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 816: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 832: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 848: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 864: 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 30 0000000000000000 +| 880: 30 30 30 30 30 30 30 30 30 05 01 01 01 02 00 00 000000000....... +| end crash-365.txt.db +}]} {} + +do_execsql_test 54.1 { + SELECT rowid, quote(matchinfo(t1,'pcxybspcxybs')) FROM t1 WHERE t1 MATCH'ATE"0"OR"2:P"""'; +} + finish_test From 02cb3ff0044a07fdefb782e375d536548e965cf5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 15 Jul 2022 20:39:39 +0000 Subject: [PATCH 075/151] In the query planner, restore the former aggressiveness in reordering of FROM clause terms that existed prior to version 3.39.0 for queries that contain no RIGHT or FULL JOINs. FossilOrigin-Name: 92d60b64ebfc2d1f0a9cabaa88e7bf0d11737ed01a77b627af10dd1b96a5321c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 14 ++++++++------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 4c513ab28f..c90ff11a61 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sin\sfts3\sthat\scould\soccur\swhen\sprocessing\sa\scorrupt\sdatabase. -D 2022-07-15T15:08:48.874 +C In\sthe\squery\splanner,\srestore\sthe\sformer\saggressiveness\sin\sreordering\sof\nFROM\sclause\sterms\sthat\sexisted\sprior\sto\sversion\s3.39.0\sfor\squeries\sthat\ncontain\sno\sRIGHT\sor\sFULL\sJOINs. +D 2022-07-15T20:39:39.491 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -657,7 +657,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c c7e96c036d2bf0755a2bedf796951ff8bf0dd062b07c4ccc724e1432c3dcf5db +F src/where.c 1049685e84bd74692ad76984a3411a21c5a1e6ddd08c981ec94d2f11f769e07f F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 F src/wherecode.c 0b09abfcb88c61c6a6984a3e065786631ff35495e9bdf865e6b74ab0a1299c5b F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 @@ -1980,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0620e419a927a3da6ebe921aaa3471686f0fdc2e485f4c2d5c88f32092228724 -R b9aac4d9609cc8841634db9ea3e3fcba -U dan -Z bb096f0e437aaa23189ce1ee6aae46bf +P d74f6f6d5136995b8bf900eb671e4b15ca81e03cc1ab5b7a1aa43dc4f3617760 +R fa54dcdbc827a4bcf8d891638157443e +U drh +Z 2feb16b88bfbd559419278240c03d9ae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 78ed53ebc3..41fe746559 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d74f6f6d5136995b8bf900eb671e4b15ca81e03cc1ab5b7a1aa43dc4f3617760 \ No newline at end of file +92d60b64ebfc2d1f0a9cabaa88e7bf0d11737ed01a77b627af10dd1b96a5321c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3dd5058fc9..6f3402a6e4 100644 --- a/src/where.c +++ b/src/where.c @@ -4181,6 +4181,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ sqlite3 *db = pWInfo->pParse->db; int rc = SQLITE_OK; int bFirstPastRJ = 0; + int hasRightJoin = 0; WhereLoop *pNew; @@ -4207,15 +4208,16 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** prevents the right operand of a RIGHT JOIN from being swapped with ** other elements even further to the right. ** - ** The JT_LTORJ term prevents any FROM-clause term reordering for terms - ** to the left of a RIGHT JOIN. This is conservative. Relaxing this - ** constraint somewhat to prevent terms from crossing from the right - ** side of a LEFT JOIN over to the left side when they are on the - ** left side of a RIGHT JOIN would be sufficient for all known failure - ** cases. FIX ME: Implement this optimization. + ** The JT_LTORJ case and the hasRightJoin flag work together to + ** prevent FROM-clause terms from moving from the right side of + ** a LEFT JOIN over to the left side of that join if the LEFT JOIN + ** is itself on the left side of a RIGHT JOIN. */ + if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1; mPrereq |= mPrior; bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0; + }else if( !hasRightJoin ){ + mPrereq = 0; } #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ From b496eef6903dd6a64a9c22611225717bb75e2371 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 16 Jul 2022 18:08:48 +0000 Subject: [PATCH 076/151] Fixes for the generated "mallocs.tcl" and "leaks.tcl" scripts generated by running tcl tests with the --malloctrace=1 option. FossilOrigin-Name: 449799e2d5902464540e8fda53ab429e0518278dab3b17c86911759114cddea0 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/malloctraceviewer.tcl | 2 +- test/tester.tcl | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c90ff11a61..7922441c62 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\squery\splanner,\srestore\sthe\sformer\saggressiveness\sin\sreordering\sof\nFROM\sclause\sterms\sthat\sexisted\sprior\sto\sversion\s3.39.0\sfor\squeries\sthat\ncontain\sno\sRIGHT\sor\sFULL\sJOINs. -D 2022-07-15T20:39:39.491 +C Fixes\sfor\sthe\sgenerated\s"mallocs.tcl"\sand\s"leaks.tcl"\sscripts\sgenerated\sby\srunning\stcl\stests\swith\sthe\s--malloctrace=1\soption. +D 2022-07-16T18:08:48.116 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1235,7 +1235,7 @@ F test/mallocK.test 25897506da0098cea09b302ff432b0fb6d8002773c1e0fc9732aa8b444bf F test/mallocL.test fb311ff80afddf3b1a75e52289081f4754d901dc F test/mallocM.test 78bbe9d3da84a5c679123cdb40d7b2010b18fc46e13897e4f253c6ba6fbff134 F test/malloc_common.tcl 806c50379cf4fa65008cd4d5af18273e5dac8ab62d1d4316c76aa2ecd2e54018 -F test/malloctraceviewer.tcl b7a54595270c1d201abf1c3f3d461f27eaf24cdef623ad08a0fe5e411264c8a9 +F test/malloctraceviewer.tcl 3e3ddf11e30d2b20f53aa16aa6615082fb24a100bea61cca7214c927b742eba6 F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 @@ -1506,7 +1506,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f1a5ce6a F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 +F test/tester.tcl d759ac44a501fb832f2ea966429ca18acfba0f9a8d34ad5c499332b079b37023 F test/testrunner.tcl 6aabdfcbfc489cc666720048606e5f9e62b8e17a3cfaf36a81fc587d9e7c39cb F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 @@ -1980,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d74f6f6d5136995b8bf900eb671e4b15ca81e03cc1ab5b7a1aa43dc4f3617760 -R fa54dcdbc827a4bcf8d891638157443e -U drh -Z 2feb16b88bfbd559419278240c03d9ae +P 92d60b64ebfc2d1f0a9cabaa88e7bf0d11737ed01a77b627af10dd1b96a5321c +R c2826cbb8b9c28cc73030d164bbfa10e +U dan +Z 0beb54d1e8fdf448050ee0b11342583e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 41fe746559..496a3b3050 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92d60b64ebfc2d1f0a9cabaa88e7bf0d11737ed01a77b627af10dd1b96a5321c \ No newline at end of file +449799e2d5902464540e8fda53ab429e0518278dab3b17c86911759114cddea0 \ No newline at end of file diff --git a/test/malloctraceviewer.tcl b/test/malloctraceviewer.tcl index 5bc22f34b3..4517fdc361 100644 --- a/test/malloctraceviewer.tcl +++ b/test/malloctraceviewer.tcl @@ -47,7 +47,7 @@ proc populate_text_widget {db} { set line [$db one {SELECT line FROM frame WHERE frame = $frame}] if {$line ne ""} { - foreach {file line} [split $line :] {} + regexp {^([^:]*):([0-9]*)} $line -> file line set content [$db one "SELECT content FROM file WHERE name = '$file'"] $::O(text) delete 0.0 end diff --git a/test/tester.tcl b/test/tester.tcl index f651828853..5612311dab 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -2171,13 +2171,13 @@ proc memdebug_log_sql {filename} { } set escaped "BEGIN; ${tbl}${tbl2}${tbl3}${sql} ; COMMIT;" - set escaped [string map [list "{" "\\{" "}" "\\}"] $escaped] + set escaped [string map [list "{" "\\{" "}" "\\}" "\\" "\\\\"] $escaped] set fd [open $filename w] puts $fd "set BUILTIN {" puts $fd $escaped puts $fd "}" - puts $fd {set BUILTIN [string map [list "\\{" "{" "\\}" "}"] $BUILTIN]} + puts $fd {set BUILTIN [string map [list "\\{" "{" "\\}" "}" "\\\\" "\\"] $BUILTIN]} set mtv [open $::testdir/malloctraceviewer.tcl] set txt [read $mtv] close $mtv From 67a0bf383f00d4f25d9ca27864caabf2a0f572b2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 18 Jul 2022 11:44:16 +0000 Subject: [PATCH 077/151] Enhance the ext/misc/regexp.c code so that when it is compiled with SQLITE_DEBUG, a new function named regexp_bytecode() is available that prints out the compiled NFA as human-readable text, for debugging purposes. FossilOrigin-Name: cb5c08978fe8f074e6ae16953575213709e98b8bbae4359e0d2e6de67a7ea9e5 --- ext/misc/regexp.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++- manifest | 14 +++---- manifest.uuid | 2 +- 3 files changed, 102 insertions(+), 9 deletions(-) diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index d5b3872058..54c505233d 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -94,6 +94,31 @@ SQLITE_EXTENSION_INIT1 #define RE_OP_NOTSPACE 16 /* Not a digit */ #define RE_OP_BOUNDARY 17 /* Boundary between word and non-word */ +#if defined(SQLITE_DEBUG) +/* Opcode names used for symbolic debugging */ +static const char *ReOpName[] = { + "EOF", + "MATCH", + "ANY", + "ANYSTAR", + "FORK", + "GOTO", + "ACCEPT", + "CC_INC", + "CC_EXC", + "CC_VALUE", + "CC_RANGE", + "WORD", + "NOTWORD", + "DIGIT", + "NOTDIGIT", + "SPACE", + "NOTSPACE", + "BOUNDARY", +}; +#endif /* SQLITE_DEBUG */ + + /* Each opcode is a "state" in the NFA */ typedef unsigned short ReStateNumber; @@ -127,7 +152,7 @@ struct ReCompiled { int *aArg; /* Arguments to each operator */ unsigned (*xNextChar)(ReInput*); /* Next character function */ unsigned char zInit[12]; /* Initial text to match */ - int nInit; /* Number of characters in zInit */ + int nInit; /* Number of bytes in zInit */ unsigned nState; /* Number of entries in aOp[] and aArg[] */ unsigned nAlloc; /* Slots allocated for aOp[] and aArg[] */ }; @@ -745,6 +770,67 @@ static void re_sql_func( } } +#if defined(SQLITE_DEBUG) +/* +** This function is used for testing and debugging only. It is only available +** if the SQLITE_DEBUG compile-time option is used. +** +** Compile a regular expression and then convert the compiled expression into +** text and return that text. +*/ +static void re_bytecode_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zPattern; + const char *zErr; + ReCompiled *pRe; + sqlite3_str *pStr; + int i; + int n; + char *z; + + zPattern = (const char*)sqlite3_value_text(argv[0]); + if( zPattern==0 ) return; + zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + if( zErr ){ + re_free(pRe); + sqlite3_result_error(context, zErr, -1); + return; + } + if( pRe==0 ){ + sqlite3_result_error_nomem(context); + return; + } + pStr = sqlite3_str_new(0); + if( pStr==0 ) goto re_bytecode_func_err; + if( pRe->nInit>0 ){ + sqlite3_str_appendf(pStr, "INIT "); + for(i=0; inInit; i++){ + sqlite3_str_appendf(pStr, "%02x", pRe->zInit[i]); + } + sqlite3_str_appendf(pStr, "\n"); + } + for(i=0; inState; i++){ + sqlite3_str_appendf(pStr, "%-8s %4d\n", + ReOpName[(unsigned char)pRe->aOp[i]], pRe->aArg[i]); + } + n = sqlite3_str_length(pStr); + z = sqlite3_str_finish(pStr); + if( n==0 ){ + sqlite3_free(z); + }else{ + sqlite3_result_text(context, z, n-1, sqlite3_free); + } + +re_bytecode_func_err: + re_free(pRe); +} + +#endif /* SQLITE_DEBUG */ + + /* ** Invoke this routine to register the regexp() function with the ** SQLite database connection. @@ -769,6 +855,13 @@ int sqlite3_regexp_init( rc = sqlite3_create_function(db, "regexpi", 2, SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, (void*)db, re_sql_func, 0, 0); +#if defined(SQLITE_DEBUG) + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "regexp_bytecode", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, re_bytecode_func, 0, 0); + } +#endif /* SQLITE_DEBUG */ } return rc; } diff --git a/manifest b/manifest index 7922441c62..e2ba6ba112 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sthe\sgenerated\s"mallocs.tcl"\sand\s"leaks.tcl"\sscripts\sgenerated\sby\srunning\stcl\stests\swith\sthe\s--malloctrace=1\soption. -D 2022-07-16T18:08:48.116 +C Enhance\sthe\sext/misc/regexp.c\scode\sso\sthat\swhen\sit\sis\scompiled\swith\nSQLITE_DEBUG,\sa\snew\sfunction\snamed\sregexp_bytecode()\sis\savailable\sthat\nprints\sout\sthe\scompiled\sNFA\sas\shuman-readable\stext,\sfor\sdebugging\spurposes. +D 2022-07-18T11:44:16.316 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -334,7 +334,7 @@ F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 F ext/misc/prefixes.c 0f4f8cff5aebc00a7e3ac4021fd59cfe1a8e17c800ceaf592859ecb9cbc38196 F ext/misc/qpvtab.c 09738419e25f603a35c0ac8bd0a04daab794f48d08a9bc07a6085b9057b99009 -F ext/misc/regexp.c c1fb4f0639ad70f180fa7ebca3b9830c1c648cdcd072e28747f727699f3a5071 +F ext/misc/regexp.c 554bbbc06980fac640618da2401b9b1a5da722f60fbc9ecd0de0244c96173a23 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 @@ -1980,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 92d60b64ebfc2d1f0a9cabaa88e7bf0d11737ed01a77b627af10dd1b96a5321c -R c2826cbb8b9c28cc73030d164bbfa10e -U dan -Z 0beb54d1e8fdf448050ee0b11342583e +P 449799e2d5902464540e8fda53ab429e0518278dab3b17c86911759114cddea0 +R c5fe187a7b0d702467f87ef01664ca12 +U drh +Z c7627cc664c2932a41026727705de830 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 496a3b3050..5a0612c077 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -449799e2d5902464540e8fda53ab429e0518278dab3b17c86911759114cddea0 \ No newline at end of file +cb5c08978fe8f074e6ae16953575213709e98b8bbae4359e0d2e6de67a7ea9e5 \ No newline at end of file From 18934137ddfd765025e73b2eb1a1bb83ee5fc03b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 18 Jul 2022 13:10:53 +0000 Subject: [PATCH 078/151] Fix a problem in the REGEXP extension for the {M,N} construct where M is zero. See [forum:/forumpost/8694e55a2c29963c|forum post 8694e55a2c29963c] for more information. FossilOrigin-Name: af15bb75306a4b94593b8431a34768b3de3d6689293e85ca02db16bf3e9f39e2 --- ext/misc/regexp.c | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- test/regexp1.test | 11 +++++++++++ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index 54c505233d..8a3e13115f 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -568,6 +568,7 @@ static const char *re_subcompile_string(ReCompiled *p){ if( m==0 ){ if( n==0 ) return "both m and n are zero in '{m,n}'"; re_insert(p, iPrev, RE_OP_FORK, sz+1); + iPrev++; n--; }else{ for(j=1; j Date: Mon, 18 Jul 2022 13:55:21 +0000 Subject: [PATCH 079/151] Enhance the REGEXP extension so that it will accept the start-of-input mark ("^") in the middle of parentheses. [forum:/forumpost/0d6a9160f81ef1a8|Forum post 0d6a9160f81ef1a8]. FossilOrigin-Name: ed8a8ebd62a319b5dabbdf67ee27141153b9899d7c8f08eeb4bdf35271015c71 --- ext/misc/regexp.c | 14 +++++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- test/regexp1.test | 14 ++++++++++++++ 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index 8a3e13115f..7413ab80ec 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -72,6 +72,7 @@ SQLITE_EXTENSION_INIT1 /* The end-of-input character */ #define RE_EOF 0 /* End of input */ +#define RE_START 0xfffffff /* Start of input - larger than an UTF-8 */ /* The NFA is implemented as sequence of opcodes taken from the following ** set. Each opcode has a single integer argument. @@ -93,6 +94,7 @@ SQLITE_EXTENSION_INIT1 #define RE_OP_SPACE 15 /* space: [ \t\n\r\v\f] */ #define RE_OP_NOTSPACE 16 /* Not a digit */ #define RE_OP_BOUNDARY 17 /* Boundary between word and non-word */ +#define RE_OP_ATSTART 18 /* Currently at the start of the string */ #if defined(SQLITE_DEBUG) /* Opcode names used for symbolic debugging */ @@ -115,6 +117,7 @@ static const char *ReOpName[] = { "SPACE", "NOTSPACE", "BOUNDARY", + "ATSTART", }; #endif /* SQLITE_DEBUG */ @@ -225,7 +228,7 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ ReStateNumber *pToFree; unsigned int i = 0; unsigned int iSwap = 0; - int c = RE_EOF+1; + int c = RE_START; int cPrev = 0; int rc = 0; ReInput in; @@ -244,6 +247,7 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ in.i++; } if( in.i+pRe->nInit>in.mx ) return 0; + c = RE_START-1; } if( pRe->nState<=(sizeof(aSpace)/(sizeof(aSpace[0])*2)) ){ @@ -272,6 +276,10 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ if( pRe->aArg[x]==c ) re_add_state(pNext, x+1); break; } + case RE_OP_ATSTART: { + if( cPrev==RE_START ) re_add_state(pThis, x+1); + break; + } case RE_OP_ANY: { if( c!=0 ) re_add_state(pNext, x+1); break; @@ -550,6 +558,10 @@ static const char *re_subcompile_string(ReCompiled *p){ re_append(p, RE_OP_MATCH, RE_EOF); break; } + case '^': { + re_append(p, RE_OP_ATSTART, 0); + break; + } case '{': { int m = 0, n = 0; int sz, j; diff --git a/manifest b/manifest index 643970b2d4..be8d9e29ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\sREGEXP\sextension\sfor\sthe\s{M,N}\sconstruct\swhere\sM\sis\szero.\nSee\s[forum:/forumpost/8694e55a2c29963c|forum\spost\s8694e55a2c29963c]\sfor\smore\ninformation. -D 2022-07-18T13:10:53.496 +C Enhance\sthe\sREGEXP\sextension\sso\sthat\sit\swill\saccept\sthe\sstart-of-input\nmark\s("^")\sin\sthe\smiddle\sof\sparentheses.\n[forum:/forumpost/0d6a9160f81ef1a8|Forum\spost\s0d6a9160f81ef1a8]. +D 2022-07-18T13:55:21.667 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -334,7 +334,7 @@ F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 F ext/misc/prefixes.c 0f4f8cff5aebc00a7e3ac4021fd59cfe1a8e17c800ceaf592859ecb9cbc38196 F ext/misc/qpvtab.c 09738419e25f603a35c0ac8bd0a04daab794f48d08a9bc07a6085b9057b99009 -F ext/misc/regexp.c 76693a7c74555947fb90bda00082b0a67ff40ac08b78c022b24623d85772fb75 +F ext/misc/regexp.c 1459fe1452b61aafb25b11d7144f3dabfaf890b566c4ef9cfa4dc270451a8f02 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 @@ -1343,7 +1343,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/recover.test ccb8c2623902a92ebb76770edd075cb4f75a4760bb7afde38026572c6e79070d -F test/regexp1.test d5d019b599d04475b1e6291ff87befeb6b6436fe8a5c96df0cd1afdd4c521860 +F test/regexp1.test 83c631617357150f8054ca1d1fed40a552b0d0f8eb7a7f090c3be02cee9f9913 F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d F test/releasetest_data.tcl 11ba48a21ed1c808147b0e77c6e93d204577f4327ffe6d7c3b34cd3c01eac3a2 @@ -1980,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb5c08978fe8f074e6ae16953575213709e98b8bbae4359e0d2e6de67a7ea9e5 -R 37f87d1133b807cf92a2fb764578dc08 +P af15bb75306a4b94593b8431a34768b3de3d6689293e85ca02db16bf3e9f39e2 +R 012943bc0a0dde589766791fd61f6489 U drh -Z e81c08e4815bfae8cfa31e84be51c0a0 +Z 6dd034c999f521ca4e2cae722ceb4800 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 65040fd41b..0d563f647d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -af15bb75306a4b94593b8431a34768b3de3d6689293e85ca02db16bf3e9f39e2 \ No newline at end of file +ed8a8ebd62a319b5dabbdf67ee27141153b9899d7c8f08eeb4bdf35271015c71 \ No newline at end of file diff --git a/test/regexp1.test b/test/regexp1.test index d7a12ceed1..102c1280c0 100644 --- a/test/regexp1.test +++ b/test/regexp1.test @@ -289,6 +289,20 @@ do_execsql_test regexp1-5.3 {SELECT 'fooX' REGEXP '^[a-z][a-z0-9]{0,2}X$';} {1} do_execsql_test regexp1-5.4 {SELECT 'foooX' REGEXP '^[a-z][a-z0-9]{0,2}X$';} {0} do_execsql_test regexp1-5.5 {SELECT 'foooX' REGEXP '^[a-z][a-z0-9]{0,3}X$';} {1} +# 2022-07-18 +# https://sqlite.org/forum/forumpost/18f87fdcdf +# Allow "^" to occur inside of "(..)" +# +do_execsql_test regexp1-6.1 {SELECT 'foo' REGEXP '[a-z]';} {1} +do_execsql_test regexp1-6.2 {SELECT 'foo' REGEXP '^[a-z]+$';} {1} +do_execsql_test regexp1-6.3 {SELECT 'foo' REGEXP '^([a-z]+)$';} {1} +do_execsql_test regexp1-6.4 {SELECT 'foo' REGEXP '(^[a-z]+)$';} {1} +do_execsql_test regexp1-6.5 {SELECT 'foo' REGEXP '(^[a-z]+$)';} {1} +do_execsql_test regexp1-6.6 {SELECT 'abc' REGEXP '(^abc|def)';} {1} +do_execsql_test regexp1-6.7 {SELECT 'xabc' REGEXP '(^abc|def)';} {0} +do_execsql_test regexp1-6.8 {SELECT 'def' REGEXP '(^abc|def)';} {1} +do_execsql_test regexp1-6.9 {SELECT 'xdef' REGEXP '(^abc|def)';} {1} + finish_test From 077e17b59a98eb8839ecfef661e7305fdb3e898d Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 18 Jul 2022 15:02:00 +0000 Subject: [PATCH 080/151] Increase the size of loop variables in the printf() implementation to avoid harmless compiler warnings. FossilOrigin-Name: aab790a16e1bdff78759f9c9ae87a2559ba82dd34ef3dedfb66035a0db7067a7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index be8d9e29ff..6b167d5434 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sREGEXP\sextension\sso\sthat\sit\swill\saccept\sthe\sstart-of-input\nmark\s("^")\sin\sthe\smiddle\sof\sparentheses.\n[forum:/forumpost/0d6a9160f81ef1a8|Forum\spost\s0d6a9160f81ef1a8]. -D 2022-07-18T13:55:21.667 +C Increase\sthe\ssize\sof\sloop\svariables\sin\sthe\sprintf()\simplementation\sto\savoid\nharmless\scompiler\swarnings. +D 2022-07-18T15:02:01.000 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -566,7 +566,7 @@ F src/pcache1.c 54881292a9a5db202b2c0ac541c5e3ef9a5e8c4f1c1383adb2601d5499a60e65 F src/pragma.c d1aead03e8418ff586c7cfca344c50a914b8eb06abd841e8e91a982d823671da F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 -F src/printf.c 6166a30417b05c5b2f82e1f183f75faa2926ad60531c0b688a57dbc951441a20 +F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c 84a8443e3723e908730d754f54df4e1dacc1eb7073c0bd79c9d5efe977a9f5b9 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 @@ -1980,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P af15bb75306a4b94593b8431a34768b3de3d6689293e85ca02db16bf3e9f39e2 -R 012943bc0a0dde589766791fd61f6489 +P ed8a8ebd62a319b5dabbdf67ee27141153b9899d7c8f08eeb4bdf35271015c71 +R 778e59f1aaa0081676e34be0c4cd31b6 U drh -Z 6dd034c999f521ca4e2cae722ceb4800 +Z 10b5a5c5b4eda184dfdf7926eb701ee8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0d563f647d..65385f089e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed8a8ebd62a319b5dabbdf67ee27141153b9899d7c8f08eeb4bdf35271015c71 \ No newline at end of file +aab790a16e1bdff78759f9c9ae87a2559ba82dd34ef3dedfb66035a0db7067a7 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index f0bfa53279..3602e1fcb9 100644 --- a/src/printf.c +++ b/src/printf.c @@ -803,8 +803,8 @@ void sqlite3_str_vappendf( case etSQLESCAPE: /* %q: Escape ' characters */ case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ case etSQLESCAPE3: { /* %w: Escape " characters */ - int i, j, k, n, isnull; - int needQuote; + i64 i, j, k, n; + int needQuote, isnull; char ch; char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */ char *escarg; From 2a7b27f1c687e87161eec84136c3efb43ac3c65d Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 18 Jul 2022 18:13:02 +0000 Subject: [PATCH 081/151] Add the "testrunner" makefile target. FossilOrigin-Name: 954c6593152f8c7372ed1233b32cce153d0ce4804869cf4ec5504d106a4920a2 --- Makefile.in | 6 ++++++ Makefile.msc | 6 ++++++ main.mk | 7 +++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/Makefile.in b/Makefile.in index 9a2ed9a183..f5b9515c10 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1281,6 +1281,12 @@ valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessio tcltest: ./testfixture$(TEXE) ./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS) +# Runs all the same tests cases as the "tcltest" target but uses +# the testrunner.tcl script to run them in multiple cores +# concurrently. +testrunner: testfixture$(TEXE) + ./testfixture$(TEXE) $(TOP)/test/testrunner.tcl + # Minimal testing that runs in less than 3 minutes # quicktest: ./testfixture$(TEXE) diff --git a/Makefile.msc b/Makefile.msc index 42da9b9e66..9baa4ae937 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2475,6 +2475,12 @@ tcltest: testfixture.exe @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS) +# Runs all the same tests cases as the "tcltest" target but uses +# the testrunner.tcl script to run them in multiple cores +# concurrently. +testrunner: testfixture.exe + .\testfixture.exe $(TOP)\test\testrunner.tcl + smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) diff --git a/main.mk b/main.mk index 35e42d6ca2..3d8a07494d 100644 --- a/main.mk +++ b/main.mk @@ -958,6 +958,12 @@ valgrindfuzz: fuzzcheck$(EXE) $(FUZZDATA) sessionfuzz$(EXE) $(TOP)/test/sessionf tcltest: ./testfixture$(EXE) ./testfixture$(EXE) $(TOP)/test/veryquick.test $(TESTOPTS) +# Runs all the same tests cases as the "tcltest" target but uses +# the testrunner.tcl script to run them in multiple cores +# concurrently. +testrunner: testfixture$(EXE) + ./testfixture$(EXE) $(TOP)/test/testrunner.tcl + # A very quick test using only testfixture and omitting all the slower # tests. Designed to run in under 3 minutes on a workstation. # @@ -968,6 +974,7 @@ quicktest: ./testfixture$(EXE) # and fuzz tests, and sqlite3_analyzer and sqldiff tests. test: fuzztest sourcetest $(TESTPROGS) tcltest + # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # diff --git a/manifest b/manifest index 6b167d5434..6557c11db1 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Increase\sthe\ssize\sof\sloop\svariables\sin\sthe\sprintf()\simplementation\sto\savoid\nharmless\scompiler\swarnings. -D 2022-07-18T15:02:01.000 +C Add\sthe\s"testrunner"\smakefile\starget. +D 2022-07-18T18:13:02.443 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 002f746b3e065e843055a23b8032b93763e68ccbaafa2b8f68532d0dfbe68deb +F Makefile.in 24d67c669635fba48c37fa5eacb01fcfde9fc522f152f60d3058c2dcf34da8da F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc de7cb3e095ce2fdc33513ccd76ebdaeda1483d0ddab0410fe65cbdeadd4c0ee1 +F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e F VERSION 8868ddfa6e1eee218286021a94b3e22d13e550c76c72d878857547ca001de24a F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -491,7 +491,7 @@ F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 50e98928657b6d5aeb9445f6130964bd644861cfa049d689db41da317d449517 +F main.mk 20801eed419dc58936ff9449b04041edbbbc0488a9fc683e72471dded050e0bb F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1980,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ed8a8ebd62a319b5dabbdf67ee27141153b9899d7c8f08eeb4bdf35271015c71 -R 778e59f1aaa0081676e34be0c4cd31b6 +P aab790a16e1bdff78759f9c9ae87a2559ba82dd34ef3dedfb66035a0db7067a7 +R b6d707203a036684fda75ff3a29c2d2f U drh -Z 10b5a5c5b4eda184dfdf7926eb701ee8 +Z 77fead86530d150ae088efbd70f2d25e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 65385f089e..ec60a5aa85 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aab790a16e1bdff78759f9c9ae87a2559ba82dd34ef3dedfb66035a0db7067a7 \ No newline at end of file +954c6593152f8c7372ed1233b32cce153d0ce4804869cf4ec5504d106a4920a2 \ No newline at end of file From cf2ad7ae3652a3fc47d3625a9bf44e5239b4d35f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 18 Jul 2022 19:32:30 +0000 Subject: [PATCH 082/151] Remove a few unsuitable scripts from the "veryquick" test suite. Also have every second testrunner.tcl process favour running test scripts that contain text like "testrunner: slow" before any others. FossilOrigin-Name: 22d280a5cd395abbedcfffbac3d3b3a614c327be25763ca380c1338a2a7bd33a --- ext/fts5/test/fts5ah.test | 1 + manifest | 43 ++++++++++++++-------------- manifest.uuid | 2 +- test/busy2.test | 1 + test/joinD.test | 2 ++ test/memjournal2.test | 1 + test/permutations.test | 4 +++ test/round1.test | 1 + test/temptable2.test | 3 ++ test/testrunner.tcl | 25 ++++++++++++++-- test/vacuum6.test | 1 + test/vacuummem.test | 1 + test/view.test | 35 ----------------------- test/view3.test | 60 +++++++++++++++++++++++++++++++++++++++ test/wal3.test | 1 + test/walsetlk.test | 2 ++ test/walvfs.test | 1 + test/win32lock.test | 2 ++ 18 files changed, 127 insertions(+), 59 deletions(-) create mode 100644 test/view3.test diff --git a/ext/fts5/test/fts5ah.test b/ext/fts5/test/fts5ah.test index 24613f5c41..0004351375 100644 --- a/ext/fts5/test/fts5ah.test +++ b/ext/fts5/test/fts5ah.test @@ -11,6 +11,7 @@ # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS5 module. # +# TESTRUNNER: slow source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ah diff --git a/manifest b/manifest index 6557c11db1..4317eedeec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"testrunner"\smakefile\starget. -D 2022-07-18T18:13:02.443 +C Remove\sa\sfew\sunsuitable\sscripts\sfrom\sthe\s"veryquick"\stest\ssuite.\sAlso\shave\severy\ssecond\stestrunner.tcl\sprocess\sfavour\srunning\stest\sscripts\sthat\scontain\stext\slike\s"testrunner:\sslow"\sbefore\sany\sothers. +D 2022-07-18T19:32:30.436 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -156,7 +156,7 @@ F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371da F ext/fts5/test/fts5ae.test 1142d16d9cc193894dc13cc8f9c7a8a21411ac61b5567a878514df6f9f0d7bb7 F ext/fts5/test/fts5af.test bea75184c0e63631b552c20ebe4a631699f357e00a2059c92538f7aeece8291e F ext/fts5/test/fts5ag.test 7816f25a0707578f08145ab539fc0ca025f8951e788b28a6a18a06b2099469dd -F ext/fts5/test/fts5ah.test 27b5a33bfd0363ca8a4dc659e6e2a5df3dea1c3c5b04bc51ca6aeb1277bd9b21 +F ext/fts5/test/fts5ah.test 2f047dfe89dc8611fa53e3d8bfc453b79cff037aa423c8d171e91e645745aa2c F ext/fts5/test/fts5ai.test bc97e4758cc93e06bf851d61c98fdf4e8b8f8315ee28a84fb15f916360856414 F ext/fts5/test/fts5aj.test 745020852d85f5dd49d11cb7ad11d3cc6dafc4fe6d6d24bc0875ac8f43ee4149 F ext/fts5/test/fts5ak.test fc3595f8e6873bb86d70c9bd4b67d0413ce577bd4793c39a2b60a7b8825b60a6 @@ -775,7 +775,7 @@ F test/btree01.test fef17d9e999ac4f04095948e3438fbe674f4e07bb2c63bb1cad41d87baee F test/btree02.test 7555a5440453d900410160a52554fe6478af4faf53098f7235f1f443d5a1d6cc F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 510dc6daaad18bcbbc085bcc6217d6dc418def5e73f72ce1475eea0cb7834727 -F test/busy2.test dbfb61b3265e7a962d3bcd32cd542bbe3d7801edbda6438d35af5aa707cae981 +F test/busy2.test 20823a5d7c42fb257d9f108c66312d90b1bb4ec3d80ba6b4e371073727560f98 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61 F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c535cfab9 @@ -1174,7 +1174,7 @@ F test/join9.test 9056ddd3b0c0f4f9d658f4521038d9a37dc23ead8ca9a505d0b0db2b6a471e F test/joinA.test 7eab225dc1c1ab258a5e62513a4ed7cabbd3db971d59d5d92f4fb6fa14c12f6a F test/joinB.test 1b2ba3fc8568b49411787fccbf540570c148e9b6a53a30f80691cb6268098ded F test/joinC.test 1f1a602c2127f55f136e2cbd3bf2d26546614bf8cffe5902ec1ac9c07f87f207 -F test/joinD.test 1a430af8dac5b68663f13df534ffe98775e582bac2305b80f1e8eb4ab778672a +F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be28 F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127 F test/joinH.test e67d1d6a8c7141caf981a07386caa7fda0362baa09e03669f9a4fbee812806d0 @@ -1241,7 +1241,7 @@ F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 F test/memdb1.test 2c4e9cc10d21c6bf4e217d72b7f6b8ba9b2605971bb2c5e6df76018e189f98f5 F test/memjournal.test 70f3a00c7f84ee2978ad14e831231caa1e7f23915a2c54b4f775a021d5740c6c -F test/memjournal2.test 89a4e0d1084170a281efa4d54c2677599f986f44227f98f7dfae282802737b65 +F test/memjournal2.test 6b9083cfaab9a3281ec545c3da2487999e8025fb7501bbae10f713f80c56454c F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 F test/memsubsys1.test 9e7555a22173b8f1c96c281ce289b338fcba2abe8b157f8798ca195bbf1d347e F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08 @@ -1318,7 +1318,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test cf5f31bab83a452288b2a050880152cdf99d62e9aab71948268d549debcc6942 +F test/permutations.test 909c84575ac50f6d30fa125a109a01986e08c26b9ea38d28501a0711b50b5627 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f F test/pragma.test cae534c12a033a5c319ccc94f50b32811acdef9f67bf19a82ff42697caccd69f F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@ -1354,7 +1354,7 @@ F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb076793 F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a -F test/round1.test 768018b04522ca420b1aba8a24bd76091d269f3bce3902af3ec6ebcee41ab21e +F test/round1.test 1bb32cf3fc505eed9e86b5e523d07e15d4428189665524587512fbcc85d114bb F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test e29025be95baf6b32f0d5edef59a7633028325896a98f1caa8019559ca910350 @@ -1503,11 +1503,11 @@ F test/tempdb.test 4cdaa23ddd8acb4d79cbb1b68ccdfd09b0537aaba909ca69a876157c2a2cb F test/tempdb2.test 353864e96fd3ae2f70773d0ffbf8b1fe48589b02c2ec05013b540879410c3440 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f1a5ce6a +F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl d759ac44a501fb832f2ea966429ca18acfba0f9a8d34ad5c499332b079b37023 -F test/testrunner.tcl 6aabdfcbfc489cc666720048606e5f9e62b8e17a3cfaf36a81fc587d9e7c39cb +F test/testrunner.tcl 86b57135754ab2160aeb04b4829d321fb285a5cfa7a505fe61d69aed605854cc F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1734,12 +1734,13 @@ F test/vacuum2.test 9fd45ce6ce29f5614c249e03938d3567c06a9e772d4f155949f8eafe2d8a F test/vacuum3.test d9d9a04ee58c485b94694fd4f68cffaba49c32234fdefe1ac1a622c5e17d4ce3 F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7c010 F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c -F test/vacuum6.test d3173a54edc81d13d99e4cf4972232b3cbb52f1d56ed48c3a939ef4e751c1ee8 -F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2 +F test/vacuum6.test b137b04bf3392d3f5c3b8fda0ce85a6775a70ca112f6559f74ff52dc9ce042fd +F test/vacuummem.test 4b30f5b95a9ff86e9d5c20741e50a898b2dc10b0962a3211571eb165357003fb F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 -F test/view.test d654fbadae82f936c2a820bbc892592085467548ff59e88acef201416e9fe48a +F test/view.test d16e49e89ada6137d1447777ef2a74574526a3b024e6733bf53ae2960da8c17c F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679 +F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -1767,7 +1768,7 @@ F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7e F test/vtabrhs1.test 9b5ecbc74a689500c33a4b2b36761f9bcc22fcc4e3f9d21066ee0c9c74cf5f6c F test/wal.test b7cc6984709f54afbf8441747ced1f646af120bf0c1b1d847bfa39306fbea089 F test/wal2.test 31f6e2c404b9f2cdf9ca19b105a1742fdc19653c2c936da39e3658c617524046 -F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 +F test/wal3.test 5de023bb862fd1eb9d2ad26fa8d9c43abb5370582e5b08b2ae0d6f93661bc310 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 F test/wal6.test b602704e4b066199bc89d91ca9000f335dcf4572 @@ -1796,11 +1797,11 @@ F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db8 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 F test/walro2.test 33955a6fd874dd9724005e17f77fef89d334b3171454a1256fe4941a96766cdc F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 -F test/walsetlk.test 3185bebc90557e0d611442c8d64f7a0cb7b06f8e156eea37a4a7358f722715be +F test/walsetlk.test c084796fc1d908957eaeba00caf85a575565be17e3333a60d5b72fe75c150387 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 -F test/walvfs.test bccb3e0d235ef85e276f491d34db32c9ada1ea67be8d9f10aabe7b30319ec656 +F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec F test/wapptest.tcl e3b6d5afa021c39a0f459ea9fbd1077459c1d81fca98eb40af8404ad3fc2360f x F test/where.test d13cd7c24e80009d2b54e2f7a8893c457afa49c64f99359c9eb3fe668ba1d9d4 @@ -1830,7 +1831,7 @@ F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f64 F test/wherelimit.test afb46397c6d7e964e6e294ba3569864a0c570fe3807afc634236c2b752372f31 F test/wherelimit2.test 657a3f24aadee62d058c5091ea682dc4af4b95ffe32f137155be49799a58e721 F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 -F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 +F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc F test/window1.test ae87c4ea4e689725c7e1c826ab2b10704d5b6d31f9b9e5abadded996ba53a1d4 @@ -1980,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aab790a16e1bdff78759f9c9ae87a2559ba82dd34ef3dedfb66035a0db7067a7 -R b6d707203a036684fda75ff3a29c2d2f -U drh -Z 77fead86530d150ae088efbd70f2d25e +P 954c6593152f8c7372ed1233b32cce153d0ce4804869cf4ec5504d106a4920a2 +R 9670f05875ab585d41bfeed2ab20423d +U dan +Z 35daffda51481f646d4c20758c239b69 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ec60a5aa85..cf901c6031 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -954c6593152f8c7372ed1233b32cce153d0ce4804869cf4ec5504d106a4920a2 \ No newline at end of file +22d280a5cd395abbedcfffbac3d3b3a614c327be25763ca380c1338a2a7bd33a \ No newline at end of file diff --git a/test/busy2.test b/test/busy2.test index 61fb8496a8..ec420c8b20 100644 --- a/test/busy2.test +++ b/test/busy2.test @@ -10,6 +10,7 @@ #*********************************************************************** # This file test the busy handler # +# TESTRUNNER: slow set testdir [file dirname $argv0] diff --git a/test/joinD.test b/test/joinD.test index c520d6a285..e4e4553b6c 100644 --- a/test/joinD.test +++ b/test/joinD.test @@ -10,6 +10,8 @@ # #*********************************************************************** # +# TESTRUNNER: slow +# # This file implements tests for JOINs that use Bloom filters. # # The test case output is (mostly) all generated by PostgreSQL 14. This diff --git a/test/memjournal2.test b/test/memjournal2.test index 97d35a98d1..ec5ba56da3 100644 --- a/test/memjournal2.test +++ b/test/memjournal2.test @@ -10,6 +10,7 @@ #*********************************************************************** # Tests focused on the in-memory journal. # +# TESTRUNNER: slow set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/permutations.test b/test/permutations.test index f2710da0f1..c5044e1a6d 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -132,6 +132,10 @@ set allquicktests [test_set $alltests -exclude { rtree4.test sessionbig.test + + writecrash.test view3.test + fts5dlidx.test fts5ac.test fts4merge3.test fts5prefix.test + sessionB.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] diff --git a/test/round1.test b/test/round1.test index ba2c79eaba..2244a399ab 100644 --- a/test/round1.test +++ b/test/round1.test @@ -10,6 +10,7 @@ #*********************************************************************** # Test cases for rounding behavior of floating point values. # +# TESTRUNNER: slow set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/temptable2.test b/test/temptable2.test index d940214495..2ee4adb9f7 100644 --- a/test/temptable2.test +++ b/test/temptable2.test @@ -8,6 +8,9 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# +# TESTRUNNER: slow +# set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/testrunner.tcl b/test/testrunner.tcl index c471473d90..da4efde267 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -58,6 +58,7 @@ set R(schema) { CREATE TABLE script( config TEXT, filename TEXT, -- full path to test script + slow BOOLEAN, -- true if script is "slow" state TEXT CHECK( state IN ('ready', 'running', 'done') ), testfixtureid, -- Id of process that ran script time INTEGER, -- Time in ms @@ -271,8 +272,23 @@ proc make_new_testset {} { db eval $R(schema) foreach t $tests { foreach {c s} $t {} + set slow 0 + + set fd [open $s] + for {set ii 0} {$ii<100 && ![eof $fd]} {incr ii} { + set line [gets $fd] + if {[string match -nocase *testrunner:* $line]} { + regexp -nocase {.*testrunner:(.*)} $line -> properties + foreach p $properties { + if {$p=="slow"} { set slow 1 } + } + } + } + close $fd + db eval { - INSERT INTO script(config, filename, state) VALUES ($c, $s, 'ready') + INSERT INTO script(config, filename, slow, state) + VALUES ($c, $s, $slow, 'ready') } } } @@ -294,7 +310,12 @@ proc get_next_test {} { set c "" db eval { SELECT config, filename FROM script WHERE state='ready' - ORDER BY config!='full', config, filename LIMIT 1 + ORDER BY + (slow * (($myid+1) % 2)) DESC, + config!='full', + config, + filename + LIMIT 1 } { set c $config set f $filename diff --git a/test/vacuum6.test b/test/vacuum6.test index e9a1542f25..f80ff75462 100644 --- a/test/vacuum6.test +++ b/test/vacuum6.test @@ -11,6 +11,7 @@ # # This file implements a test for VACUUM on attached databases. # +# TESTRUNNER: slow set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/vacuummem.test b/test/vacuummem.test index 0f77c1b400..9668b0ea90 100644 --- a/test/vacuummem.test +++ b/test/vacuummem.test @@ -12,6 +12,7 @@ # focus of this file is testing that the VACUUM statement correctly # frees any memory used for a temporary cache. # +# TESTRUNNER: slow set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/view.test b/test/view.test index 85202d7d06..b0b6da2686 100644 --- a/test/view.test +++ b/test/view.test @@ -649,41 +649,6 @@ do_test view-20.1 { } } {} -# Ticket [d58ccbb3f1b]: Prevent Table.nRef overflow. -db close -sqlite3 db :memory: -do_test view-21.1 { - catchsql { - CREATE TABLE t1(x); - INSERT INTO t1 VALUES(5); - CREATE VIEW v1 AS SELECT x*2 FROM t1; - CREATE VIEW v2 AS SELECT * FROM v1 UNION SELECT * FROM v1; - CREATE VIEW v4 AS SELECT * FROM v2 UNION SELECT * FROM v2; - CREATE VIEW v8 AS SELECT * FROM v4 UNION SELECT * FROM v4; - CREATE VIEW v16 AS SELECT * FROM v8 UNION SELECT * FROM v8; - CREATE VIEW v32 AS SELECT * FROM v16 UNION SELECT * FROM v16; - CREATE VIEW v64 AS SELECT * FROM v32 UNION SELECT * FROM v32; - CREATE VIEW v128 AS SELECT * FROM v64 UNION SELECT * FROM v64; - CREATE VIEW v256 AS SELECT * FROM v128 UNION SELECT * FROM v128; - CREATE VIEW v512 AS SELECT * FROM v256 UNION SELECT * FROM v256; - CREATE VIEW v1024 AS SELECT * FROM v512 UNION SELECT * FROM v512; - CREATE VIEW v2048 AS SELECT * FROM v1024 UNION SELECT * FROM v1024; - CREATE VIEW v4096 AS SELECT * FROM v2048 UNION SELECT * FROM v2048; - CREATE VIEW v8192 AS SELECT * FROM v4096 UNION SELECT * FROM v4096; - CREATE VIEW v16384 AS SELECT * FROM v8192 UNION SELECT * FROM v8192; - CREATE VIEW v32768 AS SELECT * FROM v16384 UNION SELECT * FROM v16384; - SELECT * FROM v32768 UNION SELECT * FROM v32768; - } -} {1 {too many references to "v1": max 65535}} -ifcapable progress { - do_test view-21.2 { - db progress 1000 {expr 1} - catchsql { - SELECT * FROM v32768; - } - } {1 interrupted} -} - db close sqlite3 db :memory: do_execsql_test view-22.1 { diff --git a/test/view3.test b/test/view3.test new file mode 100644 index 0000000000..60dd694bfd --- /dev/null +++ b/test/view3.test @@ -0,0 +1,60 @@ +# 2022 July 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing VIEW statements. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Omit this entire file if the library is not configured with views enabled. +ifcapable !view { + finish_test + return +} +set testprefix view3 + +# Ticket [d58ccbb3f1b]: Prevent Table.nRef overflow. +db close +sqlite3 db :memory: +do_test 1.1 { + catchsql { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(5); + CREATE VIEW v1 AS SELECT x*2 FROM t1; + CREATE VIEW v2 AS SELECT * FROM v1 UNION SELECT * FROM v1; + CREATE VIEW v4 AS SELECT * FROM v2 UNION SELECT * FROM v2; + CREATE VIEW v8 AS SELECT * FROM v4 UNION SELECT * FROM v4; + CREATE VIEW v16 AS SELECT * FROM v8 UNION SELECT * FROM v8; + CREATE VIEW v32 AS SELECT * FROM v16 UNION SELECT * FROM v16; + CREATE VIEW v64 AS SELECT * FROM v32 UNION SELECT * FROM v32; + CREATE VIEW v128 AS SELECT * FROM v64 UNION SELECT * FROM v64; + CREATE VIEW v256 AS SELECT * FROM v128 UNION SELECT * FROM v128; + CREATE VIEW v512 AS SELECT * FROM v256 UNION SELECT * FROM v256; + CREATE VIEW v1024 AS SELECT * FROM v512 UNION SELECT * FROM v512; + CREATE VIEW v2048 AS SELECT * FROM v1024 UNION SELECT * FROM v1024; + CREATE VIEW v4096 AS SELECT * FROM v2048 UNION SELECT * FROM v2048; + CREATE VIEW v8192 AS SELECT * FROM v4096 UNION SELECT * FROM v4096; + CREATE VIEW v16384 AS SELECT * FROM v8192 UNION SELECT * FROM v8192; + CREATE VIEW v32768 AS SELECT * FROM v16384 UNION SELECT * FROM v16384; + SELECT * FROM v32768 UNION SELECT * FROM v32768; + } +} {1 {too many references to "v1": max 65535}} +ifcapable progress { + do_test 1.2 { + db progress 1000 {expr 1} + catchsql { + SELECT * FROM v32768; + } + } {1 interrupted} +} + + +finish_test diff --git a/test/wal3.test b/test/wal3.test index 56f40ab539..cb28d0f0b9 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -12,6 +12,7 @@ # focus of this file is testing the operation of the library in # "PRAGMA journal_mode=WAL" mode. # +# TESTRUNNER: slow set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/walsetlk.test b/test/walsetlk.test index 8c0a9659f7..2f2f90ce31 100644 --- a/test/walsetlk.test +++ b/test/walsetlk.test @@ -9,6 +9,8 @@ # #*********************************************************************** # +# TESTRUNNER: slow +# set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/walvfs.test b/test/walvfs.test index d6c58656f0..d32cbd73f1 100644 --- a/test/walvfs.test +++ b/test/walvfs.test @@ -12,6 +12,7 @@ # focus of this file is testing the operation of the library in # "PRAGMA journal_mode=WAL" mode. # +# TESTRUNNER: slow set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/win32lock.test b/test/win32lock.test index d1f3d1a06e..fbb2dd13cd 100644 --- a/test/win32lock.test +++ b/test/win32lock.test @@ -12,6 +12,8 @@ # focus of this script is recovery from transient manditory locks # that sometimes appear on database files due to anti-virus software. # +# TESTRUNNER: slow +# if {$tcl_platform(platform)!="windows"} return From a5cc692422afa7fad710a4459139d7ba00346b21 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 19 Jul 2022 21:12:54 +0000 Subject: [PATCH 083/151] Improve accuracy of julian day milliseconds calculation. FossilOrigin-Name: e5e9311863544ef30dccd3bd0b3a048a864a650e69cdf9aab13dbe32a4777b51 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/date.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4317eedeec..6359633be1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sfew\sunsuitable\sscripts\sfrom\sthe\s"veryquick"\stest\ssuite.\sAlso\shave\severy\ssecond\stestrunner.tcl\sprocess\sfavour\srunning\stest\sscripts\sthat\scontain\stext\slike\s"testrunner:\sslow"\sbefore\sany\sothers. -D 2022-07-18T19:32:30.436 +C Improve\saccuracy\sof\sjulian\sday\smilliseconds\scalculation. +D 2022-07-19T21:12:54.087 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -517,7 +517,7 @@ F src/build.c 29fcc97af5197511788a571ed35a001eea472cbe3bcdbae88178e17fcafd4341 F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad -F src/date.c 15082566229d4b1e5f24fdb490bf9bcc68824b911d70e3573ef075a1b9e2d26f +F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c a8e844af211a48b13b5b358be77a12c860c6a557c21990ad51a548e2536500ce @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 954c6593152f8c7372ed1233b32cce153d0ce4804869cf4ec5504d106a4920a2 -R 9670f05875ab585d41bfeed2ab20423d -U dan -Z 35daffda51481f646d4c20758c239b69 +P 22d280a5cd395abbedcfffbac3d3b3a614c327be25763ca380c1338a2a7bd33a +R 358f17e3f8be659125138359345ba38f +U larrybr +Z 66841ff8e7be7e09568cdaf4bf69cc26 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cf901c6031..1ca07adc23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -22d280a5cd395abbedcfffbac3d3b3a614c327be25763ca380c1338a2a7bd33a \ No newline at end of file +e5e9311863544ef30dccd3bd0b3a048a864a650e69cdf9aab13dbe32a4777b51 \ No newline at end of file diff --git a/src/date.c b/src/date.c index 6e2ba202b7..68d3e994f7 100644 --- a/src/date.c +++ b/src/date.c @@ -276,7 +276,7 @@ static void computeJD(DateTime *p){ p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); p->validJD = 1; if( p->validHMS ){ - p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000); + p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5); if( p->validTZ ){ p->iJD -= p->tz*60000; p->validYMD = 0; From 3245f3be67907a31431a4506908d981ab1354523 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 20 Jul 2022 16:42:40 +0000 Subject: [PATCH 084/151] Simplify the logic that converts the "1" expression in "ORDER BY 1" into a copy of the expression that defines the first output column. FossilOrigin-Name: e1f1cfe7f4387b60443bd31742e2f49db1a2d0443200318a898ba0da216619be --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/resolve.c | 24 +++++++----------------- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 6359633be1..2197f5c054 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\saccuracy\sof\sjulian\sday\smilliseconds\scalculation. -D 2022-07-19T21:12:54.087 +C Simplify\sthe\slogic\sthat\sconverts\sthe\s"1"\sexpression\sin\s"ORDER\sBY\s1"\sinto\sa\ncopy\sof\sthe\sexpression\sthat\sdefines\sthe\sfirst\soutput\scolumn. +D 2022-07-20T16:42:40.963 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -568,7 +568,7 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c -F src/resolve.c 84a8443e3723e908730d754f54df4e1dacc1eb7073c0bd79c9d5efe977a9f5b9 +F src/resolve.c f0d663c9b1ceeb3e7d262ede872dd3b24b323a7cc11d84c05a39d962e7d64b07 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 45b93eee3c349f46240ddc14344365bbf34579ec332bd4c7bc061945e38172e2 F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 22d280a5cd395abbedcfffbac3d3b3a614c327be25763ca380c1338a2a7bd33a -R 358f17e3f8be659125138359345ba38f -U larrybr -Z 66841ff8e7be7e09568cdaf4bf69cc26 +P e5e9311863544ef30dccd3bd0b3a048a864a650e69cdf9aab13dbe32a4777b51 +R 311c30e12c0d7c0934ec40d91e93ee45 +U drh +Z d68f40fa601eed057ba107efae30da32 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1ca07adc23..a1d1130861 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e5e9311863544ef30dccd3bd0b3a048a864a650e69cdf9aab13dbe32a4777b51 \ No newline at end of file +e1f1cfe7f4387b60443bd31742e2f49db1a2d0443200318a898ba0da216619be \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 99e30d4c83..9512e3a42d 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -85,33 +85,23 @@ static void resolveAlias( sqlite3ExprDelete(db, pDup); pDup = 0; }else{ + Expr temp; incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } - - /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This - ** prevents ExprDelete() from deleting the Expr structure itself, - ** allowing it to be repopulated by the memcpy() on the following line. - ** The pExpr->u.zToken might point into memory that will be freed by the - ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to - ** make a copy of the token before doing the sqlite3DbFree(). - */ - ExprSetProperty(pExpr, EP_Static); - sqlite3ExprDelete(db, pExpr); - memcpy(pExpr, pDup, sizeof(*pExpr)); - if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){ - assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 ); - pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); - pExpr->flags |= EP_MemToken; - } + memcpy(&temp, pDup, sizeof(Expr)); + memcpy(pDup, pExpr, sizeof(Expr)); + memcpy(pExpr, &temp, sizeof(Expr)); if( ExprHasProperty(pExpr, EP_WinFunc) ){ if( ALWAYS(pExpr->y.pWin!=0) ){ pExpr->y.pWin->pOwner = pExpr; } } - sqlite3DbFree(db, pDup); + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3ExprDelete, + pDup); } } From 955301637710f18d3b44e8972f17f8cf22df4092 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 20 Jul 2022 20:36:26 +0000 Subject: [PATCH 085/151] Make use of the sqlite3ExprDeferredDelete() interface in the previous check-in, and in another place where it might be helpful. FossilOrigin-Name: 22f90e9683d5cd6619ccdb06a02e9dde9f4b7457391c0dbb4c3216c22fc0db47 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 2 +- src/resolve.c | 4 +--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 2197f5c054..28c018cc1c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\slogic\sthat\sconverts\sthe\s"1"\sexpression\sin\s"ORDER\sBY\s1"\sinto\sa\ncopy\sof\sthe\sexpression\sthat\sdefines\sthe\sfirst\soutput\scolumn. -D 2022-07-20T16:42:40.963 +C Make\suse\sof\sthe\ssqlite3ExprDeferredDelete()\sinterface\sin\sthe\sprevious\ncheck-in,\sand\sin\sanother\splace\swhere\sit\smight\sbe\shelpful. +D 2022-07-20T20:36:26.106 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c a8e844af211a48b13b5b358be77a12c860c6a557c21990ad51a548e2536500ce -F src/expr.c 4907afcb86d72b5525d8767515ce425ec53c7a2d3664441b46cef5b376ee0cba +F src/expr.c 9f568514b37dff8b1e10df6c2ccea0e5d871272d913877cde999e0168969b573 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -568,7 +568,7 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c -F src/resolve.c f0d663c9b1ceeb3e7d262ede872dd3b24b323a7cc11d84c05a39d962e7d64b07 +F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 45b93eee3c349f46240ddc14344365bbf34579ec332bd4c7bc061945e38172e2 F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e5e9311863544ef30dccd3bd0b3a048a864a650e69cdf9aab13dbe32a4777b51 -R 311c30e12c0d7c0934ec40d91e93ee45 +P e1f1cfe7f4387b60443bd31742e2f49db1a2d0443200318a898ba0da216619be +R cf76080c96baf955a2cebd7f6b61366a U drh -Z d68f40fa601eed057ba107efae30da32 +Z 4c65ef63d6c645ffaac99f031aa62bbf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a1d1130861..85bd5eafdd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1f1cfe7f4387b60443bd31742e2f49db1a2d0443200318a898ba0da216619be \ No newline at end of file +22f90e9683d5cd6619ccdb06a02e9dde9f4b7457391c0dbb4c3216c22fc0db47 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c0b2bee948..61b1d60e4b 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3325,7 +3325,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ pLimit = sqlite3PExpr(pParse, TK_NE, sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); } - sqlite3ExprDelete(db, pSel->pLimit->pLeft); + sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft); pSel->pLimit->pLeft = pLimit; }else{ /* If there is no pre-existing limit add a limit of 1 */ diff --git a/src/resolve.c b/src/resolve.c index 9512e3a42d..1c3a9d9097 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -99,9 +99,7 @@ static void resolveAlias( pExpr->y.pWin->pOwner = pExpr; } } - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprDelete, - pDup); + sqlite3ExprDeferredDelete(pParse, pDup); } } From e50998859d046da292dc2383410df67fe94a400e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 21 Jul 2022 18:37:50 +0000 Subject: [PATCH 086/151] Fix harmless compiler warning seen with MSVC. FossilOrigin-Name: 648172de20d70532ed0fb9713b76161dd481e09bbd973c03dffb51fb61b731cc --- ext/misc/regexp.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index 7413ab80ec..d0c8ee5cfe 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -825,7 +825,7 @@ static void re_bytecode_func( } sqlite3_str_appendf(pStr, "\n"); } - for(i=0; inState; i++){ + for(i=0; (unsigned)inState; i++){ sqlite3_str_appendf(pStr, "%-8s %4d\n", ReOpName[(unsigned char)pRe->aOp[i]], pRe->aArg[i]); } diff --git a/manifest b/manifest index 28c018cc1c..78ee70b915 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\suse\sof\sthe\ssqlite3ExprDeferredDelete()\sinterface\sin\sthe\sprevious\ncheck-in,\sand\sin\sanother\splace\swhere\sit\smight\sbe\shelpful. -D 2022-07-20T20:36:26.106 +C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC. +D 2022-07-21T18:37:50.752 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -334,7 +334,7 @@ F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 F ext/misc/prefixes.c 0f4f8cff5aebc00a7e3ac4021fd59cfe1a8e17c800ceaf592859ecb9cbc38196 F ext/misc/qpvtab.c 09738419e25f603a35c0ac8bd0a04daab794f48d08a9bc07a6085b9057b99009 -F ext/misc/regexp.c 1459fe1452b61aafb25b11d7144f3dabfaf890b566c4ef9cfa4dc270451a8f02 +F ext/misc/regexp.c 5abed0ace2d9340b42b9ab1dbe64db9c276e4e8eba38a903232b6253e05ccdaf F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e1f1cfe7f4387b60443bd31742e2f49db1a2d0443200318a898ba0da216619be -R cf76080c96baf955a2cebd7f6b61366a -U drh -Z 4c65ef63d6c645ffaac99f031aa62bbf +P 22f90e9683d5cd6619ccdb06a02e9dde9f4b7457391c0dbb4c3216c22fc0db47 +R 846ac44e12ef98f830013180b827ef68 +U mistachkin +Z 35c5050a4917217d748070c1e316ab4b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 85bd5eafdd..af6fbce23c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -22f90e9683d5cd6619ccdb06a02e9dde9f4b7457391c0dbb4c3216c22fc0db47 \ No newline at end of file +648172de20d70532ed0fb9713b76161dd481e09bbd973c03dffb51fb61b731cc \ No newline at end of file From d00505dc97e030f7b68bd631db14c27128f82a51 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 22 Jul 2022 14:52:29 +0000 Subject: [PATCH 087/151] Update the documentation for SQLITE_OPEN_NOFOLLOW to state more clearly that nothing in the database path is allowed to be a symbolic link. FossilOrigin-Name: de9222697b2ca8cae2cc7aa9082cca3910038ebbab1e8ee15270b06711711a0b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 78ee70b915..41afa25e23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC. -D 2022-07-21T18:37:50.752 +C Update\sthe\sdocumentation\sfor\sSQLITE_OPEN_NOFOLLOW\sto\sstate\smore\sclearly\sthat\nnothing\sin\sthe\sdatabase\spath\sis\sallowed\sto\sbe\sa\ssymbolic\slink. +D 2022-07-22T14:52:29.510 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -572,7 +572,7 @@ F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 45b93eee3c349f46240ddc14344365bbf34579ec332bd4c7bc061945e38172e2 F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 -F src/sqlite.h.in 01573eae96721f2a8ee2a9e3b7140ceeba2e9c44350911890b89b8ff0dcf6781 +F src/sqlite.h.in be265991edca9aea69986758b58ba81cbf5ae403fe0c4ea1d0c9df0cdc8f25ed F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d F src/sqliteInt.h 059d5a017ebf488c7484f79ea507f56b2bf4bb700f340abf91c5d2227869f275 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 22f90e9683d5cd6619ccdb06a02e9dde9f4b7457391c0dbb4c3216c22fc0db47 -R 846ac44e12ef98f830013180b827ef68 -U mistachkin -Z 35c5050a4917217d748070c1e316ab4b +P 648172de20d70532ed0fb9713b76161dd481e09bbd973c03dffb51fb61b731cc +R 22d3a3c7680400f82d17fa9e3cf5e084 +U drh +Z ef2231159a73a46fc3d5cbe83757be79 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index af6fbce23c..bd99275ee3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -648172de20d70532ed0fb9713b76161dd481e09bbd973c03dffb51fb61b731cc \ No newline at end of file +de9222697b2ca8cae2cc7aa9082cca3910038ebbab1e8ee15270b06711711a0b \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index e2281e4978..f6b49b4d88 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3439,7 +3439,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** to return an extended result code. ** ** [[OPEN_NOFOLLOW]] ^(
[SQLITE_OPEN_NOFOLLOW]
-**
The database filename is not allowed to be a symbolic link
+**
The database filename is not allowed to contain a symbolic link
** )^ ** ** If the 3rd parameter to sqlite3_open_v2() is not one of the From 034d11180796267db921a5faba2d98ec6d73fe48 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 22 Jul 2022 18:25:04 +0000 Subject: [PATCH 088/151] In a TK_BLOB Expr node, the Expr.zToken might not be a well-formed BLOB literal if there has been a prior OOM. dbsqlfuzz 23871e5805d6c45b392f9b7aa1e8a2b98f3c27cd. FossilOrigin-Name: c538d075350927222ab0a6598b844f7b15153c5dc008d71b921a2b73c4f4a7a4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 41afa25e23..416917b7f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sdocumentation\sfor\sSQLITE_OPEN_NOFOLLOW\sto\sstate\smore\sclearly\sthat\nnothing\sin\sthe\sdatabase\spath\sis\sallowed\sto\sbe\sa\ssymbolic\slink. -D 2022-07-22T14:52:29.510 +C In\sa\sTK_BLOB\sExpr\snode,\sthe\sExpr.zToken\smight\snot\sbe\sa\swell-formed\nBLOB\sliteral\sif\sthere\shas\sbeen\sa\sprior\sOOM.\ndbsqlfuzz\s23871e5805d6c45b392f9b7aa1e8a2b98f3c27cd. +D 2022-07-22T18:25:04.652 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c a8e844af211a48b13b5b358be77a12c860c6a557c21990ad51a548e2536500ce -F src/expr.c 9f568514b37dff8b1e10df6c2ccea0e5d871272d913877cde999e0168969b573 +F src/expr.c 10b3e1a052ccadbb81037e273bd4482831dcebdbbab379d895df655a5ff7e305 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 648172de20d70532ed0fb9713b76161dd481e09bbd973c03dffb51fb61b731cc -R 22d3a3c7680400f82d17fa9e3cf5e084 +P de9222697b2ca8cae2cc7aa9082cca3910038ebbab1e8ee15270b06711711a0b +R 6f602e521caa17d238271eb589ae4e8e U drh -Z ef2231159a73a46fc3d5cbe83757be79 +Z 19001449740bd3515437c61d096c4c20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bd99275ee3..09f38cfb08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de9222697b2ca8cae2cc7aa9082cca3910038ebbab1e8ee15270b06711711a0b \ No newline at end of file +c538d075350927222ab0a6598b844f7b15153c5dc008d71b921a2b73c4f4a7a4 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 61b1d60e4b..93e2f1e8ef 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4212,6 +4212,7 @@ expr_code_doover: int n; const char *z; char *zBlob; + if( pParse->nErr ) return target; assert( !ExprHasProperty(pExpr, EP_IntValue) ); assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); assert( pExpr->u.zToken[1]=='\'' ); From 825fa17b1f220e9ba45757251becdde41958ae8e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 22 Jul 2022 19:28:04 +0000 Subject: [PATCH 089/151] Omit the EP_MemToken flag that was made obsolete by [e1f1cfe7f4387b60], for a size reduction and performance increase. FossilOrigin-Name: 28934a9d92d5e5ac862a0dc7169f071f39047f98dc79441db697cf353a4b9433 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 7 +------ src/sqliteInt.h | 2 +- src/window.c | 1 - 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 416917b7f1..ddd8d2b466 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sa\sTK_BLOB\sExpr\snode,\sthe\sExpr.zToken\smight\snot\sbe\sa\swell-formed\nBLOB\sliteral\sif\sthere\shas\sbeen\sa\sprior\sOOM.\ndbsqlfuzz\s23871e5805d6c45b392f9b7aa1e8a2b98f3c27cd. -D 2022-07-22T18:25:04.652 +C Omit\sthe\sEP_MemToken\sflag\sthat\swas\smade\sobsolete\sby\s[e1f1cfe7f4387b60],\sfor\na\ssize\sreduction\sand\sperformance\sincrease. +D 2022-07-22T19:28:04.965 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c a8e844af211a48b13b5b358be77a12c860c6a557c21990ad51a548e2536500ce -F src/expr.c 10b3e1a052ccadbb81037e273bd4482831dcebdbbab379d895df655a5ff7e305 +F src/expr.c 44f6b019a581f7c5a6599464d76d19111427edfa2710b4f007968fc64aaed0d1 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -575,7 +575,7 @@ F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d3 F src/sqlite.h.in be265991edca9aea69986758b58ba81cbf5ae403fe0c4ea1d0c9df0cdc8f25ed F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d -F src/sqliteInt.h 059d5a017ebf488c7484f79ea507f56b2bf4bb700f340abf91c5d2227869f275 +F src/sqliteInt.h c97b6351dba36b7bacd231610cd7173e7a7ef8469c2d9b849037415e0b1d8c79 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4a3da6d77eeb3531cb0dbdf7047772a2a1b99f98c69e90ce009c75fe6328b2c0 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -661,7 +661,7 @@ F src/where.c 1049685e84bd74692ad76984a3411a21c5a1e6ddd08c981ec94d2f11f769e07f F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 F src/wherecode.c 0b09abfcb88c61c6a6984a3e065786631ff35495e9bdf865e6b74ab0a1299c5b F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 -F src/window.c fff1b51757438c664e471d5184634e48dcdf8ea34b640f3b1b0810b1e06de18c +F src/window.c 3409e5067705ad8b858475e36fc22dc342192c02158d2b7c5219b9c1f96aaf55 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test b5c19d504dec222c0dc66642673d23dce915d35737b68e74d9f237b80493eb53 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P de9222697b2ca8cae2cc7aa9082cca3910038ebbab1e8ee15270b06711711a0b -R 6f602e521caa17d238271eb589ae4e8e +P c538d075350927222ab0a6598b844f7b15153c5dc008d71b921a2b73c4f4a7a4 +R 0a28921b7e90ddf823369af11249c300 U drh -Z 19001449740bd3515437c61d096c4c20 +Z 8e5ad3553ebf08b44289f99a5ac2ab7d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 09f38cfb08..15e2267724 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c538d075350927222ab0a6598b844f7b15153c5dc008d71b921a2b73c4f4a7a4 \ No newline at end of file +28934a9d92d5e5ac862a0dc7169f071f39047f98dc79441db697cf353a4b9433 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 93e2f1e8ef..3801fb98f4 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1238,10 +1238,6 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ #endif } } - if( ExprHasProperty(p, EP_MemToken) ){ - assert( !ExprHasProperty(p, EP_IntValue) ); - sqlite3DbFree(db, p->u.zToken); - } if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbFreeNN(db, p); } @@ -1349,7 +1345,6 @@ static int dupedExprStructSize(const Expr *p, int flags){ }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); assert( !ExprHasProperty(p, EP_OuterON) ); - assert( !ExprHasProperty(p, EP_MemToken) ); assert( !ExprHasVVAProperty(p, EP_NoReduce) ); if( p->pLeft || p->x.pList ){ nSize = EXPR_REDUCEDSIZE | EP_Reduced; @@ -1453,7 +1448,7 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ } /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ - pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static); pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); pNew->flags |= staticFlag; ExprClearVVAProperties(pNew); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 106a5d1475..c6bdd3dd35 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2886,7 +2886,7 @@ struct Expr { #define EP_Reduced 0x004000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ #define EP_Win 0x008000 /* Contains window functions */ #define EP_TokenOnly 0x010000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ -#define EP_MemToken 0x020000 /* Need to sqlite3DbFree() Expr.zToken */ + /* 0x020000 // Available for reuse */ #define EP_IfNullRow 0x040000 /* The TK_IF_NULL_ROW opcode */ #define EP_Unlikely 0x080000 /* unlikely() or likelihood() function */ #define EP_ConstFunc 0x100000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ diff --git a/src/window.c b/src/window.c index 893668664f..cb7681f461 100644 --- a/src/window.c +++ b/src/window.c @@ -900,7 +900,6 @@ static ExprList *exprListAppendList( for(i=0; inExpr; i++){ sqlite3 *db = pParse->db; Expr *pDup = sqlite3ExprDup(db, pAppend->a[i].pExpr, 0); - assert( pDup==0 || !ExprHasProperty(pDup, EP_MemToken) ); if( db->mallocFailed ){ sqlite3ExprDelete(db, pDup); break; From ee6b80c3b50efb65630584eb9001d2771055ff8d Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 23 Jul 2022 00:44:44 +0000 Subject: [PATCH 090/151] Use sqlite3ParserAddCleanup() rather than pParse->pConstExpr to implement sqlite3ExprDeferredDelete(). This is a better solution than check-in [c538d07535092722]. FossilOrigin-Name: 2a6f6971fa010219323b976ff53b2606e39fb43fd36c394837c3a8528bf3c425 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ddd8d2b466..07ebf75ee4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\sEP_MemToken\sflag\sthat\swas\smade\sobsolete\sby\s[e1f1cfe7f4387b60],\sfor\na\ssize\sreduction\sand\sperformance\sincrease. -D 2022-07-22T19:28:04.965 +C Use\ssqlite3ParserAddCleanup()\srather\sthan\spParse->pConstExpr\sto\simplement\nsqlite3ExprDeferredDelete().\s\sThis\sis\sa\sbetter\ssolution\sthan\scheck-in\s\n[c538d07535092722]. +D 2022-07-23T00:44:44.539 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c a8e844af211a48b13b5b358be77a12c860c6a557c21990ad51a548e2536500ce -F src/expr.c 44f6b019a581f7c5a6599464d76d19111427edfa2710b4f007968fc64aaed0d1 +F src/expr.c 78a1b6c13306efaf563e9340732a2d651f792c488b7795a25f1f6a853e07ef25 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c538d075350927222ab0a6598b844f7b15153c5dc008d71b921a2b73c4f4a7a4 -R 0a28921b7e90ddf823369af11249c300 +P 28934a9d92d5e5ac862a0dc7169f071f39047f98dc79441db697cf353a4b9433 +R 88cc25988976487ed2e20df47dba68d4 U drh -Z 8e5ad3553ebf08b44289f99a5ac2ab7d +Z fe526c3fdeb2c1220cb11dfe559b4150 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 15e2267724..f97e721aa1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28934a9d92d5e5ac862a0dc7169f071f39047f98dc79441db697cf353a4b9433 \ No newline at end of file +2a6f6971fa010219323b976ff53b2606e39fb43fd36c394837c3a8528bf3c425 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 3801fb98f4..8edab3298a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1270,8 +1270,9 @@ void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ ** pExpr to the pParse->pConstExpr list with a register number of 0. */ void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ - pParse->pConstExpr = - sqlite3ExprListAppend(pParse, pParse->pConstExpr, pExpr); + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3ExprDelete, + pExpr); } /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the @@ -4207,7 +4208,6 @@ expr_code_doover: int n; const char *z; char *zBlob; - if( pParse->nErr ) return target; assert( !ExprHasProperty(pExpr, EP_IntValue) ); assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); assert( pExpr->u.zToken[1]=='\'' ); From 3dedb87bd79822948234862154699f1e7781999a Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 23 Jul 2022 00:53:48 +0000 Subject: [PATCH 091/151] Remove a branch that is no longer reachable due to the previous check-in. FossilOrigin-Name: 8b4d1b9317624f443fe526858bfe3da5281ba83de1828c62935f48b3e7cf2a88 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/build.c | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 07ebf75ee4..3578f3ff26 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\ssqlite3ParserAddCleanup()\srather\sthan\spParse->pConstExpr\sto\simplement\nsqlite3ExprDeferredDelete().\s\sThis\sis\sa\sbetter\ssolution\sthan\scheck-in\s\n[c538d07535092722]. -D 2022-07-23T00:44:44.539 +C Remove\sa\sbranch\sthat\sis\sno\slonger\sreachable\sdue\sto\sthe\sprevious\scheck-in. +D 2022-07-23T00:53:48.586 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -513,7 +513,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 1307d57f65023c9f37c2b6c62253343fca63877f161f694a593b14c0883cedda F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e -F src/build.c 29fcc97af5197511788a571ed35a001eea472cbe3bcdbae88178e17fcafd4341 +F src/build.c 149663ba16fc023d3ee1929d2fac40e0055617cd58e49d4e75b99376ddd40bd3 F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 28934a9d92d5e5ac862a0dc7169f071f39047f98dc79441db697cf353a4b9433 -R 88cc25988976487ed2e20df47dba68d4 +P 2a6f6971fa010219323b976ff53b2606e39fb43fd36c394837c3a8528bf3c425 +R 620499f569e880513867b67bc9090f80 U drh -Z fe526c3fdeb2c1220cb11dfe559b4150 +Z 0dba3b3710f22c731b61e5cb97b7e027 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f97e721aa1..2f48cc1742 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2a6f6971fa010219323b976ff53b2606e39fb43fd36c394837c3a8528bf3c425 \ No newline at end of file +8b4d1b9317624f443fe526858bfe3da5281ba83de1828c62935f48b3e7cf2a88 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 31ab81b09b..b48876ac52 100644 --- a/src/build.c +++ b/src/build.c @@ -260,9 +260,7 @@ void sqlite3FinishCoding(Parse *pParse){ pParse->okConstFactor = 0; for(i=0; inExpr; i++){ int iReg = pEL->a[i].u.iConstExprReg; - if( iReg>0 ){ - sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); - } + sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); } } From 7bace9e9bb3d2b67b5a3e5e567b7be2cd9cc1e97 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 23 Jul 2022 12:51:48 +0000 Subject: [PATCH 092/151] Simplifications to sqlite3FinishCoding() for a small size reduction and performance increase. FossilOrigin-Name: a995614b9aedf4492e6d7b777293770f268837f8246e1678ef0523738c8a8339 --- manifest | 12 ++--- manifest.uuid | 2 +- src/build.c | 118 ++++++++++++++++++++++++-------------------------- 3 files changed, 64 insertions(+), 68 deletions(-) diff --git a/manifest b/manifest index 3578f3ff26..2e7a764e4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sbranch\sthat\sis\sno\slonger\sreachable\sdue\sto\sthe\sprevious\scheck-in. -D 2022-07-23T00:53:48.586 +C Simplifications\sto\ssqlite3FinishCoding()\sfor\sa\ssmall\ssize\sreduction\sand\nperformance\sincrease. +D 2022-07-23T12:51:48.615 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -513,7 +513,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 1307d57f65023c9f37c2b6c62253343fca63877f161f694a593b14c0883cedda F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e -F src/build.c 149663ba16fc023d3ee1929d2fac40e0055617cd58e49d4e75b99376ddd40bd3 +F src/build.c 249fb44843805cded2003e0e678ab65acb98ae0982e23256d4d0a7576a440eb3 F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2a6f6971fa010219323b976ff53b2606e39fb43fd36c394837c3a8528bf3c425 -R 620499f569e880513867b67bc9090f80 +P 8b4d1b9317624f443fe526858bfe3da5281ba83de1828c62935f48b3e7cf2a88 +R 4e3eaba4ea506cda95e5ca36886afbe8 U drh -Z 0dba3b3710f22c731b61e5cb97b7e027 +Z 03b9158b57b304bce795274f430799f1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2f48cc1742..d3e71676b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b4d1b9317624f443fe526858bfe3da5281ba83de1828c62935f48b3e7cf2a88 \ No newline at end of file +a995614b9aedf4492e6d7b777293770f268837f8246e1678ef0523738c8a8339 \ No newline at end of file diff --git a/src/build.c b/src/build.c index b48876ac52..df33c493a4 100644 --- a/src/build.c +++ b/src/build.c @@ -140,6 +140,7 @@ int sqlite3DbMaskAllZero(yDbMask m){ void sqlite3FinishCoding(Parse *pParse){ sqlite3 *db; Vdbe *v; + int iDb, i; assert( pParse->pToplevel==0 ); db = pParse->db; @@ -206,74 +207,69 @@ void sqlite3FinishCoding(Parse *pParse){ ** transaction on each used database and to verify the schema cookie ** on each used database. */ - if( db->mallocFailed==0 - && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) - ){ - int iDb, i; - assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); - sqlite3VdbeJumpHere(v, 0); - assert( db->nDb>0 ); - iDb = 0; - do{ - Schema *pSchema; - if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; - sqlite3VdbeUsesBtree(v, iDb); - pSchema = db->aDb[iDb].pSchema; - sqlite3VdbeAddOp4Int(v, - OP_Transaction, /* Opcode */ - iDb, /* P1 */ - DbMaskTest(pParse->writeMask,iDb), /* P2 */ - pSchema->schema_cookie, /* P3 */ - pSchema->iGeneration /* P4 */ - ); - if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); - VdbeComment((v, - "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); - }while( ++iDbnDb ); + assert( pParse->nErr>0 || sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); + sqlite3VdbeJumpHere(v, 0); + assert( db->nDb>0 ); + iDb = 0; + do{ + Schema *pSchema; + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; + sqlite3VdbeUsesBtree(v, iDb); + pSchema = db->aDb[iDb].pSchema; + sqlite3VdbeAddOp4Int(v, + OP_Transaction, /* Opcode */ + iDb, /* P1 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ + pSchema->schema_cookie, /* P3 */ + pSchema->iGeneration /* P4 */ + ); + if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); + }while( ++iDbnDb ); #ifndef SQLITE_OMIT_VIRTUALTABLE - for(i=0; inVtabLock; i++){ - char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); - sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); - } - pParse->nVtabLock = 0; + for(i=0; inVtabLock; i++){ + char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); + } + pParse->nVtabLock = 0; #endif - /* Once all the cookies have been verified and transactions opened, - ** obtain the required table-locks. This is a no-op unless the - ** shared-cache feature is enabled. - */ - codeTableLocks(pParse); + /* Once all the cookies have been verified and transactions opened, + ** obtain the required table-locks. This is a no-op unless the + ** shared-cache feature is enabled. + */ + codeTableLocks(pParse); - /* Initialize any AUTOINCREMENT data structures required. - */ - sqlite3AutoincrementBegin(pParse); + /* Initialize any AUTOINCREMENT data structures required. + */ + sqlite3AutoincrementBegin(pParse); - /* Code constant expressions that where factored out of inner loops. - ** - ** The pConstExpr list might also contain expressions that we simply - ** want to keep around until the Parse object is deleted. Such - ** expressions have iConstExprReg==0. Do not generate code for - ** those expressions, of course. - */ - if( pParse->pConstExpr ){ - ExprList *pEL = pParse->pConstExpr; - pParse->okConstFactor = 0; - for(i=0; inExpr; i++){ - int iReg = pEL->a[i].u.iConstExprReg; - sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); - } + /* Code constant expressions that where factored out of inner loops. + ** + ** The pConstExpr list might also contain expressions that we simply + ** want to keep around until the Parse object is deleted. Such + ** expressions have iConstExprReg==0. Do not generate code for + ** those expressions, of course. + */ + if( pParse->pConstExpr ){ + ExprList *pEL = pParse->pConstExpr; + pParse->okConstFactor = 0; + for(i=0; inExpr; i++){ + int iReg = pEL->a[i].u.iConstExprReg; + sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); } - - if( pParse->bReturning ){ - Returning *pRet = pParse->u1.pReturning; - if( pRet->nRetCol ){ - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); - } - } - - /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeGoto(v, 1); } + + if( pParse->bReturning ){ + Returning *pRet = pParse->u1.pReturning; + if( pRet->nRetCol ){ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); + } + } + + /* Finally, jump back to the beginning of the executable code. */ + sqlite3VdbeGoto(v, 1); } /* Get the VDBE program ready for execution From 8bd0b250b715802f3bddf46596841d18acffcdb4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Jul 2022 11:04:13 +0000 Subject: [PATCH 093/151] Reduce a timeout in walsetlk.test from 2000ms to 1100ms so that the test runs a bit faster. FossilOrigin-Name: 836fa097060dadeb2dc5d4ee2e40621c4af606b1ef7241e2264823e23e4ceb1f --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/walsetlk.test | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2e7a764e4f..c38bca3b5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplifications\sto\ssqlite3FinishCoding()\sfor\sa\ssmall\ssize\sreduction\sand\nperformance\sincrease. -D 2022-07-23T12:51:48.615 +C Reduce\sa\stimeout\sin\swalsetlk.test\sfrom\s2000ms\sto\s1100ms\sso\sthat\sthe\stest\sruns\sa\sbit\sfaster. +D 2022-07-25T11:04:13.326 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1797,7 +1797,7 @@ F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db8 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 F test/walro2.test 33955a6fd874dd9724005e17f77fef89d334b3171454a1256fe4941a96766cdc F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 -F test/walsetlk.test c084796fc1d908957eaeba00caf85a575565be17e3333a60d5b72fe75c150387 +F test/walsetlk.test 34c901443b31ab720afc463f5b236c86ca5c4134402573dce91aa0761de8db5a F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8b4d1b9317624f443fe526858bfe3da5281ba83de1828c62935f48b3e7cf2a88 -R 4e3eaba4ea506cda95e5ca36886afbe8 -U drh -Z 03b9158b57b304bce795274f430799f1 +P a995614b9aedf4492e6d7b777293770f268837f8246e1678ef0523738c8a8339 +R e7b8638d6eb2bb65a2c18035ce7a719d +U dan +Z 34c5d50caba419fff3085e94b713365e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d3e71676b3..69076e5991 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a995614b9aedf4492e6d7b777293770f268837f8246e1678ef0523738c8a8339 \ No newline at end of file +836fa097060dadeb2dc5d4ee2e40621c4af606b1ef7241e2264823e23e4ceb1f \ No newline at end of file diff --git a/test/walsetlk.test b/test/walsetlk.test index 2f2f90ce31..1e09238226 100644 --- a/test/walsetlk.test +++ b/test/walsetlk.test @@ -88,7 +88,7 @@ do_multiclient_test tn { INSERT INTO t1 VALUES(3, 4); INSERT INTO t1 VALUES(5, 6); } - code1 { db timeout 2000 } + code1 { db timeout 1100 } } {} do_test 2.$tn.2 { From 6b6d6c6bd2ca6b1dc19f61216ffe88e5cf734581 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Jul 2022 14:05:11 +0000 Subject: [PATCH 094/151] TK_IF_NULL_ROW expressions must be accumulated in the same way as TK_COLUMN expressions in an aggregate query. Proposed fix for the problem identifyed by dbsqlfuzz 8e17857db2c5a9294c975123ac807156a6559f13. FossilOrigin-Name: 40d08807209638aad728be2cedbc904e342e76c8e486c364bd571b55dd2e1e87 --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/expr.c | 10 +++++++++- test/select3.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c38bca3b5e..d040b26a47 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sa\stimeout\sin\swalsetlk.test\sfrom\s2000ms\sto\s1100ms\sso\sthat\sthe\stest\sruns\sa\sbit\sfaster. -D 2022-07-25T11:04:13.326 +C TK_IF_NULL_ROW\sexpressions\smust\sbe\saccumulated\sin\sthe\ssame\sway\sas\sTK_COLUMN\nexpressions\sin\san\saggregate\squery.\s\sProposed\sfix\sfor\sthe\sproblem\sidentifyed\sby\ndbsqlfuzz\s8e17857db2c5a9294c975123ac807156a6559f13. +D 2022-07-25T14:05:11.599 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c a8e844af211a48b13b5b358be77a12c860c6a557c21990ad51a548e2536500ce -F src/expr.c 78a1b6c13306efaf563e9340732a2d651f792c488b7795a25f1f6a853e07ef25 +F src/expr.c 72f312252a5791988242dc812df89778f53939e3f60f4e0b432e90f4a3081c1d F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -1391,7 +1391,7 @@ F test/securedel.test 2f70b2449186a1921bd01ec9da407fbfa98c3a7a5521854c300c194b2f F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/select1.test 692e84cfa29c405854c69e8a4027183d64c22952866a123fabbce741a379e889 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 -F test/select3.test ce4f78bbc809b0513f960f1ee84cdbc5af50ba112c343d5266558a8b2468f656 +F test/select3.test 054b155a4b9394c6858640029cb93e87defbaecc1c87ebb21157c3d35dfc4d88 F test/select4.test f0684d3da3bccacbe2a1ebadf6fb49d9df6f53acb4c6ebc228a88d0d6054cc7b F test/select5.test 8afc5e5dcdebc2be54472e73ebd9cd1adef1225fd15d37a1c62f969159f390ae F test/select6.test 9b2fb4ffedf52e1b5703cfcae1212e7a4a063f014c0458d78d29aca3db766d1f @@ -1981,8 +1981,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a995614b9aedf4492e6d7b777293770f268837f8246e1678ef0523738c8a8339 -R e7b8638d6eb2bb65a2c18035ce7a719d -U dan -Z 34c5d50caba419fff3085e94b713365e +P 836fa097060dadeb2dc5d4ee2e40621c4af606b1ef7241e2264823e23e4ceb1f +R ef7a712abb1795061ef7b33ca931bdb8 +T *branch * flatten-left-join +T *sym-flatten-left-join * +T -sym-trunk * +U drh +Z 4c1e333b8e885220474e580d77e5a634 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 69076e5991..383392c434 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -836fa097060dadeb2dc5d4ee2e40621c4af606b1ef7241e2264823e23e4ceb1f \ No newline at end of file +40d08807209638aad728be2cedbc904e342e76c8e486c364bd571b55dd2e1e87 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 8edab3298a..876b453f16 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4670,6 +4670,13 @@ expr_code_doover: case TK_IF_NULL_ROW: { int addrINR; u8 okConstFactor = pParse->okConstFactor; + if( pExpr->pAggInfo && !pExpr->pAggInfo->directMode ){ + struct AggInfo_col *pCol; + assert( pExpr->iAgg>=0 && pExpr->iAggpAggInfo->nColumn ); + pCol = &pExpr->pAggInfo->aCol[pExpr->iAgg]; + inReg = pCol->iMem; + break; + } addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); /* Temporarily disable factoring of constant expressions, since ** even though expressions may appear to be constant, they are not @@ -6175,6 +6182,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ assert( pNC->ncFlags & NC_UAggInfo ); switch( pExpr->op ){ + case TK_IF_NULL_ROW: case TK_AGG_COLUMN: case TK_COLUMN: { testcase( pExpr->op==TK_AGG_COLUMN ); @@ -6237,7 +6245,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ */ ExprSetVVAProperty(pExpr, EP_NoReduce); pExpr->pAggInfo = pAggInfo; - pExpr->op = TK_AGG_COLUMN; + if( pExpr->op==TK_COLUMN ) pExpr->op = TK_AGG_COLUMN; pExpr->iAgg = (i16)k; break; } /* endif pExpr->iTable==pItem->iCursor */ diff --git a/test/select3.test b/test/select3.test index 809b549028..ec0ee8d2bb 100644 --- a/test/select3.test +++ b/test/select3.test @@ -330,4 +330,49 @@ do_execsql_test select3.10.100 { FROM t1; } {{} {}} +#------------------------------------------------------------------------- +# dbsqlfuzz crash-8e17857db2c5a9294c975123ac807156a6559f13.txt +# Associated with the flatten-left-join branch circa 2022-06-23. +# +foreach {tn sql} { + 1 { + CREATE TABLE t1(a TEXT); + CREATE TABLE t2(x INT); + CREATE INDEX t2x ON t2(x); + INSERT INTO t1 VALUES('abc'); + } + 2 { + CREATE TABLE t1(a TEXT); + CREATE TABLE t2(x INT); + INSERT INTO t1 VALUES('abc'); + } + 3 { + CREATE TABLE t1(a TEXT); + CREATE TABLE t2(x INT); + INSERT INTO t1 VALUES('abc'); + PRAGMA automatic_index=OFF; + } +} { + reset_db + do_execsql_test select3-11.$tn.1 $sql + do_execsql_test select3.11.$tn.2 { + SELECT max(a), val FROM t1 LEFT JOIN ( + SELECT 'constant' AS val FROM t2 WHERE x=1234 + ) + } {abc {}} + do_execsql_test select3.11.$tn.3 { + INSERT INTO t2 VALUES(123); + SELECT max(a), val FROM t1 LEFT JOIN ( + SELECT 'constant' AS val FROM t2 WHERE x=1234 + ) + } {abc {}} + do_execsql_test select3.11.$tn.4 { + INSERT INTO t2 VALUES(1234); + SELECT max(a), val FROM t1 LEFT JOIN ( + SELECT 'constant' AS val FROM t2 WHERE x=1234 + ) + } {abc constant} +} + + finish_test From ee37302095f95b8692d835dc3dec4cbb398d9c3b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Jul 2022 15:54:23 +0000 Subject: [PATCH 095/151] Allow subqueries on the right-hand side of a LEFT JOIN to be flattened even if they contain a GROUP BY clause. FossilOrigin-Name: 816da9a893ae97a21463562479edb419a8b511ae731d86eccee3fa6e3e7dc96e --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/expr.c | 25 ++++++++++++++++++------- src/select.c | 11 +++++------ 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index d040b26a47..a5d100a729 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C TK_IF_NULL_ROW\sexpressions\smust\sbe\saccumulated\sin\sthe\ssame\sway\sas\sTK_COLUMN\nexpressions\sin\san\saggregate\squery.\s\sProposed\sfix\sfor\sthe\sproblem\sidentifyed\sby\ndbsqlfuzz\s8e17857db2c5a9294c975123ac807156a6559f13. -D 2022-07-25T14:05:11.599 +C Allow\ssubqueries\son\sthe\sright-hand\sside\sof\sa\sLEFT\sJOIN\sto\sbe\sflattened\seven\nif\sthey\scontain\sa\sGROUP\sBY\sclause. +D 2022-07-25T15:54:23.818 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c a8e844af211a48b13b5b358be77a12c860c6a557c21990ad51a548e2536500ce -F src/expr.c 72f312252a5791988242dc812df89778f53939e3f60f4e0b432e90f4a3081c1d +F src/expr.c 61681ff95f4017181b975cdf2790dcbcda6ec962786041286bf5a8b6e051e2ca F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -570,7 +570,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 45b93eee3c349f46240ddc14344365bbf34579ec332bd4c7bc061945e38172e2 +F src/select.c a9516e1453241986f1eb73c00a0e7cf23d23081fd2eb50e67fcbb96fe7bf6f00 F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 F src/sqlite.h.in be265991edca9aea69986758b58ba81cbf5ae403fe0c4ea1d0c9df0cdc8f25ed F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1981,11 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 836fa097060dadeb2dc5d4ee2e40621c4af606b1ef7241e2264823e23e4ceb1f -R ef7a712abb1795061ef7b33ca931bdb8 -T *branch * flatten-left-join -T *sym-flatten-left-join * -T -sym-trunk * +P 40d08807209638aad728be2cedbc904e342e76c8e486c364bd571b55dd2e1e87 +R 9e57786ca604cfa312cb51673a055fc9 U drh -Z 4c1e333b8e885220474e580d77e5a634 +Z fc9a3eeded753547b019835285a3d918 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 383392c434..790aa223e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -40d08807209638aad728be2cedbc904e342e76c8e486c364bd571b55dd2e1e87 \ No newline at end of file +816da9a893ae97a21463562479edb419a8b511ae731d86eccee3fa6e3e7dc96e \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 876b453f16..c9c7c2e760 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4670,12 +4670,20 @@ expr_code_doover: case TK_IF_NULL_ROW: { int addrINR; u8 okConstFactor = pParse->okConstFactor; - if( pExpr->pAggInfo && !pExpr->pAggInfo->directMode ){ - struct AggInfo_col *pCol; - assert( pExpr->iAgg>=0 && pExpr->iAggpAggInfo->nColumn ); - pCol = &pExpr->pAggInfo->aCol[pExpr->iAgg]; - inReg = pCol->iMem; - break; + AggInfo *pAggInfo = pExpr->pAggInfo; + if( pAggInfo ){ + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + if( !pAggInfo->directMode ){ + inReg = pAggInfo->aCol[pExpr->iAgg].iMem; + break; + } + if( pExpr->pAggInfo->useSortingIdx ){ + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, + pAggInfo->aCol[pExpr->iAgg].iSorterColumn, + target); + inReg = target; + break; + } } addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); /* Temporarily disable factoring of constant expressions, since @@ -6187,6 +6195,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ case TK_COLUMN: { testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_COLUMN ); + testcase( pExpr->op==TK_IF_NULL_ROW ); /* Check to see if the column is in one of the tables in the FROM ** clause of the aggregate query */ if( ALWAYS(pSrcList!=0) ){ @@ -6245,7 +6254,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ */ ExprSetVVAProperty(pExpr, EP_NoReduce); pExpr->pAggInfo = pAggInfo; - if( pExpr->op==TK_COLUMN ) pExpr->op = TK_AGG_COLUMN; + if( pExpr->op==TK_COLUMN ){ + pExpr->op = TK_AGG_COLUMN; + } pExpr->iAgg = (i16)k; break; } /* endif pExpr->iTable==pItem->iCursor */ diff --git a/src/select.c b/src/select.c index 90ba47fa8a..672461d447 100644 --- a/src/select.c +++ b/src/select.c @@ -4065,8 +4065,8 @@ static void renumberCursors( ** (3a) the subquery may not be a join and ** (3b) the FROM clause of the subquery may not contain a virtual ** table and -** (3c) The outer query may not have a GROUP BY. (This limitation is -** due to how TK_IF_NULL_ROW works. FIX ME!) +** (**) Was: "The outer query may not have a GROUP BY." This case +** is now managed correctly ** (3d) the outer query may not be DISTINCT. ** See also (26) for restrictions on RIGHT JOIN. ** @@ -4284,7 +4284,6 @@ static int flattenSubquery( if( pSubSrc->nSrc>1 /* (3a) */ || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ - || (p->pGroupBy!=0) /* (3c) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ ){ return 0; @@ -7541,15 +7540,15 @@ int sqlite3Select( regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; + pAggInfo->directMode = 1; for(i=0; inColumn; i++){ struct AggInfo_col *pCol = &pAggInfo->aCol[i]; if( pCol->iSorterColumn>=j ){ - int r1 = j + regBase; - sqlite3ExprCodeGetColumnOfTable(v, - pCol->pTab, pCol->iTable, pCol->iColumn, r1); + sqlite3ExprCode(pParse, pCol->pCExpr, j + regBase); j++; } } + pAggInfo->directMode = 0; regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); From 058e99502a70abb5e67bead9a4146f4523ff89d9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Jul 2022 19:05:24 +0000 Subject: [PATCH 096/151] Small performance increase and size reduction by splitting out the sqlite3VdbeGetLastOp() from sqlite3VdbeGetOp(). FossilOrigin-Name: 92ac01d41d46ab73e189b1e5596ea63e5edb5b15639c5d7bdb981b95366c069b --- manifest | 27 +++++++++++++-------------- manifest.uuid | 2 +- src/delete.c | 5 +++-- src/expr.c | 6 +++--- src/insert.c | 6 +++--- src/pragma.c | 2 +- src/vdbe.h | 1 + src/vdbeaux.c | 23 +++++++++++++++-------- src/wherecode.c | 2 +- src/window.c | 6 ++---- 10 files changed, 43 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 40e6111774..8d9ac4ab45 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sthe\saggregate\squery\sLEFT\sJOIN\sflattening\soptimization\sfrom\n[2cf373b10c9bc4cb]\sand\sfurther\senhance\sthat\soptimization\sso\sthat\sit\sworks\neven\sif\sthere\sis\sa\sGROUP\sBY\sclause. -D 2022-07-25T16:06:14.582 +C Small\sperformance\sincrease\sand\ssize\sreduction\sby\ssplitting\sout\sthe\nsqlite3VdbeGetLastOp()\sfrom\ssqlite3VdbeGetOp(). +D 2022-07-25T19:05:24.840 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,8 +520,8 @@ F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d -F src/delete.c a8e844af211a48b13b5b358be77a12c860c6a557c21990ad51a548e2536500ce -F src/expr.c 61681ff95f4017181b975cdf2790dcbcda6ec962786041286bf5a8b6e051e2ca +F src/delete.c 13eca2beee5b758ed033a11230971310cc4a58fcd8f6bc33cad4f677c985e96c +F src/expr.c 375b8285b347a33ea239eb1dad378d4044fe5a6b0d9ee1e7db7370f97d5fb14e F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -530,7 +530,7 @@ F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 173845e5a6bac96ae937409e4f876b631f26b31dabb9df8fd0eb3b130b2bb3a7 +F src/insert.c ffbfe57fb73c0f14b30ce8a5da431519cdd315e20a9dd07fa12ae8e526d37f49 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b @@ -563,7 +563,7 @@ F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564 F src/pcache.c 084e638432c610f95aea72b8509f0845d2791293f39d1b82f0c0a7e089c3bb6b F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 54881292a9a5db202b2c0ac541c5e3ef9a5e8c4f1c1383adb2601d5499a60e65 -F src/pragma.c d1aead03e8418ff586c7cfca344c50a914b8eb06abd841e8e91a982d823671da +F src/pragma.c 6637d624c37a8909d3edfa9d7cf694d79b49d2a0827d8c52ef15dceb641783fa F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 @@ -643,10 +643,10 @@ F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 602fe229f32a96ceccae4f40824129669582096f7c355f53dbac156c9fecef23 F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 F src/vdbe.c 1266f3a4744224253dd74f0080014be8056b062c6f2f6a81e229fa0d306d4102 -F src/vdbe.h 07641758ca8b4f4c6d81ea667ea167c541e6ece21f5574da11e3d21ec37e2662 +F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c d68267db6e6641994e17c70670c40fd67ceb2352e42188815ed8c05d4d6502cb -F src/vdbeaux.c 444c399df547e003be52cc51b460fed3b63e1f18939e6b773ff99c584954b726 +F src/vdbeaux.c f14f30892ee8491f7903cf4ccd7bd0b7ab6f1eddf7dda4a0e03725fcd5ae05e1 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 5ebf05c0182addedb1607ade848e1c83cef40981df94d1abfab0c59288c6064f F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -659,9 +659,9 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 1049685e84bd74692ad76984a3411a21c5a1e6ddd08c981ec94d2f11f769e07f F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 -F src/wherecode.c 0b09abfcb88c61c6a6984a3e065786631ff35495e9bdf865e6b74ab0a1299c5b +F src/wherecode.c 210240c9cec2a1d1494a3ac7852e9432245382e752b490c2bb437637c2c98711 F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 -F src/window.c 3409e5067705ad8b858475e36fc22dc342192c02158d2b7c5219b9c1f96aaf55 +F src/window.c 057df98b3b8296c4ccd35066ee171b66d956b3ee8bfce27596ddb5ed8fab66f4 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test b5c19d504dec222c0dc66642673d23dce915d35737b68e74d9f237b80493eb53 @@ -1981,9 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 836fa097060dadeb2dc5d4ee2e40621c4af606b1ef7241e2264823e23e4ceb1f 816da9a893ae97a21463562479edb419a8b511ae731d86eccee3fa6e3e7dc96e -R 9e57786ca604cfa312cb51673a055fc9 -T +closed 816da9a893ae97a21463562479edb419a8b511ae731d86eccee3fa6e3e7dc96e +P b52393ac28debe9867227f901d05cccf54f1b467272474500a549d956a5fb4d7 +R d4f59dff4c0149f1eec71fecf2aaf224 U drh -Z c715e74c49fbf8938da5e0c5419cf34a +Z da216fd20b1c63c1a0740d95d247a9a9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b115932a65..e1b00675ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b52393ac28debe9867227f901d05cccf54f1b467272474500a549d956a5fb4d7 \ No newline at end of file +92ac01d41d46ab73e189b1e5596ea63e5edb5b15639c5d7bdb981b95366c069b \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index df378d2d58..3d0e055be2 100644 --- a/src/delete.c +++ b/src/delete.c @@ -447,9 +447,10 @@ void sqlite3DeleteFrom( } for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - sqlite3VdbeChangeP3(v, -1, memCnt ? memCnt : -1); + sqlite3VdbeAddOp3(v, OP_Clear, pIdx->tnum, iDb, memCnt ? memCnt : -1); + }else{ + sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); } } }else diff --git a/src/expr.c b/src/expr.c index c9c7c2e760..c383f6f65f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3835,7 +3835,7 @@ int sqlite3ExprCodeGetColumn( assert( pParse->pVdbe!=0 ); sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); if( p5 ){ - VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + VdbeOp *pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); if( pOp->opcode==OP_Column ) pOp->p5 = p5; } return iReg; @@ -3904,7 +3904,7 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ ** so that a subsequent copy will not be merged into this one. */ static void setDoNotMergeFlagOnCopy(Vdbe *v){ - if( sqlite3VdbeGetOp(v, -1)->opcode==OP_Copy ){ + if( sqlite3VdbeGetLastOp(v)->opcode==OP_Copy ){ sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergable */ } } @@ -5026,7 +5026,7 @@ int sqlite3ExprCodeExprList( if( inReg!=target+i ){ VdbeOp *pOp; if( copyOp==OP_Copy - && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy + && (pOp=sqlite3VdbeGetLastOp(v))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg && pOp->p2+pOp->p3+1==target+i && pOp->p5==0 /* The do-not-merge flag must be clear */ diff --git a/src/insert.c b/src/insert.c index 9b97b99a35..507bc017e2 100644 --- a/src/insert.c +++ b/src/insert.c @@ -159,7 +159,7 @@ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ ** OP_MakeRecord is found */ VdbeOp *pPrev; sqlite3VdbeAppendP4(v, pTab, P4_TABLE); - pPrev = sqlite3VdbeGetOp(v, -1); + pPrev = sqlite3VdbeGetLastOp(v); assert( pPrev!=0 ); assert( pPrev->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); pPrev->opcode = OP_TypeCheck; @@ -197,7 +197,7 @@ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ if( iReg ){ sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); }else{ - assert( sqlite3VdbeGetOp(v, -1)->opcode==OP_MakeRecord + assert( sqlite3VdbeGetLastOp(v)->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); sqlite3VdbeChangeP4(v, -1, zColAff, i); } @@ -283,7 +283,7 @@ void sqlite3ComputeGeneratedColumns( */ sqlite3TableAffinity(pParse->pVdbe, pTab, iRegStore); if( (pTab->tabFlags & TF_HasStored)!=0 ){ - pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); if( pOp->opcode==OP_Affinity ){ /* Change the OP_Affinity argument to '@' (NONE) for all stored ** columns. '@' is the no-op affinity and those columns have not diff --git a/src/pragma.c b/src/pragma.c index 9860da86d7..0a1bc37fae 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1781,7 +1781,7 @@ void sqlite3Pragma( if( pCol->notNull==0 && !bStrict ) continue; doError = bStrict ? sqlite3VdbeMakeLabel(pParse) : 0; sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); - if( sqlite3VdbeGetOp(v,-1)->opcode==OP_Column ){ + if( sqlite3VdbeGetLastOp(v)->opcode==OP_Column ){ sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); } if( pCol->notNull ){ diff --git a/src/vdbe.h b/src/vdbe.h index 5909d3995d..eb1445f1db 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -242,6 +242,7 @@ void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type); void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); void sqlite3VdbeUsesBtree(Vdbe*, int); VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); +VdbeOp *sqlite3VdbeGetLastOp(Vdbe*); int sqlite3VdbeMakeLabel(Parse*); void sqlite3VdbeRunOnlyOnce(Vdbe*); void sqlite3VdbeReusable(Vdbe*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 4666f728ab..dfee43d05c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -456,7 +456,7 @@ void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ iThis = v->nOp; sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, zMsg, P4_DYNAMIC); - sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetOp(v,-1)->p4.z); + sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetLastOp(v)->p4.z); if( bPush){ pParse->addrExplain = iThis; } @@ -1133,15 +1133,19 @@ void sqlite3VdbeScanStatus( ** for a specific instruction. */ void sqlite3VdbeChangeOpcode(Vdbe *p, int addr, u8 iNewOpcode){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; } void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p1 = val; } void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p2 = val; } void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p3 = val; } void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ @@ -1177,7 +1181,7 @@ void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ || p->aOp[addr].opcode==OP_FkIfZero ); assert( p->aOp[addr].p4type==0 ); #ifdef SQLITE_VDBE_COVERAGE - sqlite3VdbeGetOp(p,-1)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ + sqlite3VdbeGetLastOp(p)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ #endif p->nOp--; }else{ @@ -1498,13 +1502,13 @@ void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ ** Set the value if the iSrcLine field for the previously coded instruction. */ void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ - sqlite3VdbeGetOp(v,-1)->iSrcLine = iLine; + sqlite3VdbeGetLastOp(v)->iSrcLine = iLine; } #endif /* SQLITE_VDBE_COVERAGE */ /* -** Return the opcode for a given address. If the address is -1, then -** return the most recently inserted opcode. +** Return the opcode for a given address. The address must be non-negative. +** See sqlite3VdbeGetLastOp() to get the most recently added opcode. ** ** If a memory allocation error has occurred prior to the calling of this ** routine, then a pointer to a dummy VdbeOp will be returned. That opcode @@ -1520,9 +1524,6 @@ VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ ** zeros, which is correct. MSVC generates a warning, nevertheless. */ static VdbeOp dummy; /* Ignore the MSVC warning about no initializer */ assert( p->eVdbeState==VDBE_INIT_STATE ); - if( addr<0 ){ - addr = p->nOp - 1; - } assert( (addr>=0 && addrnOp) || p->db->mallocFailed ); if( p->db->mallocFailed ){ return (VdbeOp*)&dummy; @@ -1531,6 +1532,12 @@ VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ } } +/* Return the most recently added opcode +*/ +VdbeOp * sqlite3VdbeGetLastOp(Vdbe *p){ + return sqlite3VdbeGetOp(p, p->nOp - 1); +} + #if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) /* ** Return an integer value for one of the parameters to the opcode pOp diff --git a/src/wherecode.c b/src/wherecode.c index 04f1c374d0..db7c60ec2a 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -892,7 +892,7 @@ static void whereLikeOptimizationStringFixup( if( pTerm->wtFlags & TERM_LIKEOPT ){ VdbeOp *pOp; assert( pLevel->iLikeRepCntr>0 ); - pOp = sqlite3VdbeGetOp(v, -1); + pOp = sqlite3VdbeGetLastOp(v); assert( pOp!=0 ); assert( pOp->opcode==OP_String8 || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); diff --git a/src/window.c b/src/window.c index cb7681f461..030365851c 100644 --- a/src/window.c +++ b/src/window.c @@ -2170,10 +2170,8 @@ static void windowCodeRangeTest( /* This block runs if reg1 is not NULL, but reg2 is. */ sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); VdbeCoverage(v); - if( op==OP_Gt || op==OP_Ge ){ - sqlite3VdbeChangeP2(v, -1, addrDone); - } + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, + (op==OP_Gt || op==OP_Ge) ? addrDone : lbl); } /* Register reg1 currently contains csr1.peerVal (the peer-value from csr1). From 47e2fe3ce78af0c2dd32a0e8f6e10d2211a932d2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Jul 2022 20:21:57 +0000 Subject: [PATCH 097/151] Performance optimization in computing the Expr.nHeight field. FossilOrigin-Name: 1798ce97c8763d75315e1716d10f6c5be301042c174f41ee8c1fb8d9db99d52b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 17 +++++++++++++++-- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8d9ac4ab45..3d1d1a278e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\sincrease\sand\ssize\sreduction\sby\ssplitting\sout\sthe\nsqlite3VdbeGetLastOp()\sfrom\ssqlite3VdbeGetOp(). -D 2022-07-25T19:05:24.840 +C Performance\soptimization\sin\scomputing\sthe\sExpr.nHeight\sfield. +D 2022-07-25T20:21:57.291 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 13eca2beee5b758ed033a11230971310cc4a58fcd8f6bc33cad4f677c985e96c -F src/expr.c 375b8285b347a33ea239eb1dad378d4044fe5a6b0d9ee1e7db7370f97d5fb14e +F src/expr.c afc33c8b4f72a61e49194aa2c6f475e4e7ba87e91a5102701d02978d292000e2 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b52393ac28debe9867227f901d05cccf54f1b467272474500a549d956a5fb4d7 -R d4f59dff4c0149f1eec71fecf2aaf224 +P 92ac01d41d46ab73e189b1e5596ea63e5edb5b15639c5d7bdb981b95366c069b +R 5546500321041b9016b316ba858300f9 U drh -Z da216fd20b1c63c1a0740d95d247a9a9 +Z dddc2675e82e808f6c9034d235b5fa31 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e1b00675ab..e9a838e5a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92ac01d41d46ab73e189b1e5596ea63e5edb5b15639c5d7bdb981b95366c069b \ No newline at end of file +1798ce97c8763d75315e1716d10f6c5be301042c174f41ee8c1fb8d9db99d52b \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c383f6f65f..44744524ab 100644 --- a/src/expr.c +++ b/src/expr.c @@ -770,7 +770,9 @@ static void heightOfSelect(const Select *pSelect, int *pnHeight){ */ static void exprSetHeight(Expr *p){ int nHeight = p->pLeft ? p->pLeft->nHeight : 0; - if( p->pRight && p->pRight->nHeight>nHeight ) nHeight = p->pRight->nHeight; + if( NEVER(p->pRight) && p->pRight->nHeight>nHeight ){ + nHeight = p->pRight->nHeight; + } if( ExprUseXSelect(p) ){ heightOfSelect(p->x.pSelect, &nHeight); }else if( p->x.pList ){ @@ -913,15 +915,26 @@ void sqlite3ExprAttachSubtrees( sqlite3ExprDelete(db, pLeft); sqlite3ExprDelete(db, pRight); }else{ + assert( ExprUseXList(pRoot) ); + assert( pRoot->x.pSelect==0 ); if( pRight ){ pRoot->pRight = pRight; pRoot->flags |= EP_Propagate & pRight->flags; +#if SQLITE_MAX_EXPR_DEPTH>0 + pRoot->nHeight = pRight->nHeight+1; + }else{ + pRoot->nHeight = 1; +#endif } if( pLeft ){ pRoot->pLeft = pLeft; pRoot->flags |= EP_Propagate & pLeft->flags; +#if SQLITE_MAX_EXPR_DEPTH>0 + if( pLeft->nHeight>=pRoot->nHeight ){ + pRoot->nHeight = pLeft->nHeight+1; + } +#endif } - exprSetHeight(pRoot); } } From 2d2e528e1f46e326c0c9e6fe27d47fab3b93c945 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Jul 2022 21:37:13 +0000 Subject: [PATCH 098/151] In-line a call to sqlite3ExprCode() in insert.c, for a size reduction and performance increase. FossilOrigin-Name: 35066b1446228bf030795e7868509c7b54a5681984ac28bf43123f8fac2e361e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/insert.c | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3d1d1a278e..aabe381def 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\scomputing\sthe\sExpr.nHeight\sfield. -D 2022-07-25T20:21:57.291 +C In-line\sa\scall\sto\ssqlite3ExprCode()\sin\sinsert.c,\sfor\sa\ssize\sreduction\sand\nperformance\sincrease. +D 2022-07-25T21:37:13.343 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -530,7 +530,7 @@ F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c ffbfe57fb73c0f14b30ce8a5da431519cdd315e20a9dd07fa12ae8e526d37f49 +F src/insert.c 0dad9475a958e9d3bfd6c4bc3507b5ed0ec7cca615c423c652b19e2676d59b3f F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 92ac01d41d46ab73e189b1e5596ea63e5edb5b15639c5d7bdb981b95366c069b -R 5546500321041b9016b316ba858300f9 +P 1798ce97c8763d75315e1716d10f6c5be301042c174f41ee8c1fb8d9db99d52b +R 893e05cda15fed15bec15ef6790fcb7b U drh -Z dddc2675e82e808f6c9034d235b5fa31 +Z b54185c467fd683a9e6941abfcb36a89 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e9a838e5a8..6aa0dd4538 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1798ce97c8763d75315e1716d10f6c5be301042c174f41ee8c1fb8d9db99d52b \ No newline at end of file +35066b1446228bf030795e7868509c7b54a5681984ac28bf43123f8fac2e361e \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 507bc017e2..91004fb470 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1189,7 +1189,12 @@ void sqlite3Insert( sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore); } }else{ - sqlite3ExprCode(pParse, pList->a[k].pExpr, iRegStore); + Expr *pX = pList->a[k].pExpr; + int y = sqlite3ExprCodeTarget(pParse, pX, iRegStore); + if( y!=iRegStore ){ + sqlite3VdbeAddOp2(v, + ExprHasProperty(pX, EP_Subquery) ? OP_Copy : OP_SCopy, y, iRegStore); + } } } From f89812fe526d29db71fd4181d5efc40b4ced2648 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Jul 2022 22:02:35 +0000 Subject: [PATCH 099/151] Fix a harmless compiler warning. FossilOrigin-Name: 9aba7417c60b30b44c051580f0f997a4d2fc86bc811b532d175305ad4c181fc6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/build.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index aabe381def..5d003aaa49 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In-line\sa\scall\sto\ssqlite3ExprCode()\sin\sinsert.c,\sfor\sa\ssize\sreduction\sand\nperformance\sincrease. -D 2022-07-25T21:37:13.343 +C Fix\sa\sharmless\scompiler\swarning. +D 2022-07-25T22:02:35.792 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -513,7 +513,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 1307d57f65023c9f37c2b6c62253343fca63877f161f694a593b14c0883cedda F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e -F src/build.c 249fb44843805cded2003e0e678ab65acb98ae0982e23256d4d0a7576a440eb3 +F src/build.c abba8a3f8529d3f3808e7cfa580e4541471ea010635d2a23365c0de8b22128b5 F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1798ce97c8763d75315e1716d10f6c5be301042c174f41ee8c1fb8d9db99d52b -R 893e05cda15fed15bec15ef6790fcb7b +P 35066b1446228bf030795e7868509c7b54a5681984ac28bf43123f8fac2e361e +R 14bc2153dd870c51d01ef34ec6804518 U drh -Z b54185c467fd683a9e6941abfcb36a89 +Z 26d5ce22a72ed03f342cb02915325d2e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6aa0dd4538..269b8d0bb9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35066b1446228bf030795e7868509c7b54a5681984ac28bf43123f8fac2e361e \ No newline at end of file +9aba7417c60b30b44c051580f0f997a4d2fc86bc811b532d175305ad4c181fc6 \ No newline at end of file diff --git a/src/build.c b/src/build.c index df33c493a4..4b27d5cf84 100644 --- a/src/build.c +++ b/src/build.c @@ -170,7 +170,6 @@ void sqlite3FinishCoding(Parse *pParse){ if( pParse->bReturning ){ Returning *pReturning = pParse->u1.pReturning; int addrRewind; - int i; int reg; if( pReturning->nRetCol ){ From ef69d2b277c545fa8fe193ab9ec87b96ccbd05f7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Jul 2022 22:31:04 +0000 Subject: [PATCH 100/151] Performance optimization in sqlite3ViewGetColumnNames(). FossilOrigin-Name: 390717e68800af9b71acd635cf6cb123f9a591276df511f11462b42960f9a70c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/build.c | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5d003aaa49..dfa107fe51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning. -D 2022-07-25T22:02:35.792 +C Performance\soptimization\sin\ssqlite3ViewGetColumnNames(). +D 2022-07-25T22:31:04.172 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -513,7 +513,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 1307d57f65023c9f37c2b6c62253343fca63877f161f694a593b14c0883cedda F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e -F src/build.c abba8a3f8529d3f3808e7cfa580e4541471ea010635d2a23365c0de8b22128b5 +F src/build.c 941d7ff0c38f8b16ae5ffff8d76a7657701115a42651d9e52054c12ef0a6ad79 F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 35066b1446228bf030795e7868509c7b54a5681984ac28bf43123f8fac2e361e -R 14bc2153dd870c51d01ef34ec6804518 +P 9aba7417c60b30b44c051580f0f997a4d2fc86bc811b532d175305ad4c181fc6 +R 6455264c445f081e2167b0aeb515b69f U drh -Z 26d5ce22a72ed03f342cb02915325d2e +Z 96e54ff79e297fd32468d545082b5401 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 269b8d0bb9..cdf20ffd4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9aba7417c60b30b44c051580f0f997a4d2fc86bc811b532d175305ad4c181fc6 \ No newline at end of file +390717e68800af9b71acd635cf6cb123f9a591276df511f11462b42960f9a70c \ No newline at end of file diff --git a/src/build.c b/src/build.c index 4b27d5cf84..47aa8bc57c 100644 --- a/src/build.c +++ b/src/build.c @@ -3042,7 +3042,7 @@ create_view_fail: ** the columns of the view in the pTable structure. Return the number ** of errors. If an error is seen leave an error message in pParse->zErrMsg. */ -int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ +static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ Select *pSel; /* Copy of the SELECT that implements the view */ int nErr = 0; /* Number of errors encountered */ @@ -3165,6 +3165,11 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #endif /* SQLITE_OMIT_VIEW */ return nErr; } +int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ + assert( pTable!=0 ); + if( !IsVirtual(pTable) && pTable->nCol>0 ) return 0; + return viewGetColumnNames(pParse, pTable); +} #endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ #ifndef SQLITE_OMIT_VIEW From 509a6303d140d00068f5c33ededb91191934a434 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Jul 2022 23:01:41 +0000 Subject: [PATCH 101/151] Performance optimization by only invoking sqlite3FkCheck() when it is actually needed. FossilOrigin-Name: 98b0e830bc7effa3c9cbb77aebe7c128afc3b210af336b7516108d6435705ba0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 5 +++-- src/insert.c | 4 +++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index dfa107fe51..220822117c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\ssqlite3ViewGetColumnNames(). -D 2022-07-25T22:31:04.172 +C Performance\soptimization\sby\sonly\sinvoking\ssqlite3FkCheck()\swhen\sit\sis\sactually\nneeded. +D 2022-07-25T23:01:41.122 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -513,7 +513,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 1307d57f65023c9f37c2b6c62253343fca63877f161f694a593b14c0883cedda F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e -F src/build.c 941d7ff0c38f8b16ae5ffff8d76a7657701115a42651d9e52054c12ef0a6ad79 +F src/build.c 33aec2b34dbc53c3da2ea8434fccbf3c17c35ba647c3193df598cce0e861355c F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad @@ -530,7 +530,7 @@ F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 0dad9475a958e9d3bfd6c4bc3507b5ed0ec7cca615c423c652b19e2676d59b3f +F src/insert.c a8c994e3c3b8b08f61745bedabdf5affc79584a2b3c80ee2e4f038817838bd0a F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9aba7417c60b30b44c051580f0f997a4d2fc86bc811b532d175305ad4c181fc6 -R 6455264c445f081e2167b0aeb515b69f +P 390717e68800af9b71acd635cf6cb123f9a591276df511f11462b42960f9a70c +R fad65d0eefcf9a391d6bfd56620c6c0d U drh -Z 96e54ff79e297fd32468d545082b5401 +Z 0b33966f147041d5e60fb39f3e8d2b14 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cdf20ffd4d..7e63185ea7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -390717e68800af9b71acd635cf6cb123f9a591276df511f11462b42960f9a70c \ No newline at end of file +98b0e830bc7effa3c9cbb77aebe7c128afc3b210af336b7516108d6435705ba0 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 47aa8bc57c..db69b611fa 100644 --- a/src/build.c +++ b/src/build.c @@ -3067,9 +3067,10 @@ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ #ifndef SQLITE_OMIT_VIEW /* A positive nCol means the columns names for this view are - ** already known. + ** already known. This routine is not called unless either the + ** table is virtual or nCol is zero. */ - if( pTable->nCol>0 ) return 0; + assert( pTable->nCol<=0 ); /* A negative nCol is a special marker meaning that we are currently ** trying to compute the column names. If we enter this routine with diff --git a/src/insert.c b/src/insert.c index 91004fb470..6de304848e 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1331,7 +1331,9 @@ void sqlite3Insert( sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert ); - sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + if( db->flags & SQLITE_ForeignKeys ){ + sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + } /* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE ** constraints or (b) there are no triggers and this table is not a From fe888bcf1173dbcd4f21be0e2dba747973cb6e35 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Jul 2022 23:34:14 +0000 Subject: [PATCH 102/151] Fix an assert() that was made out-of-date by [b52393ac28debe98]. FossilOrigin-Name: db8230e80ed894fc2801cdbb53c2cd9de00192d7f4d8328c29cc214e2e0adde9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 220822117c..af92ee66bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sby\sonly\sinvoking\ssqlite3FkCheck()\swhen\sit\sis\sactually\nneeded. -D 2022-07-25T23:01:41.122 +C Fix\san\sassert()\sthat\swas\smade\sout-of-date\sby\s[b52393ac28debe98]. +D 2022-07-25T23:34:14.245 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 13eca2beee5b758ed033a11230971310cc4a58fcd8f6bc33cad4f677c985e96c -F src/expr.c afc33c8b4f72a61e49194aa2c6f475e4e7ba87e91a5102701d02978d292000e2 +F src/expr.c 1565349664995e930cc11217cc665875d8d3d3cae62c92805a67ab582befc92a F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 390717e68800af9b71acd635cf6cb123f9a591276df511f11462b42960f9a70c -R fad65d0eefcf9a391d6bfd56620c6c0d +P 98b0e830bc7effa3c9cbb77aebe7c128afc3b210af336b7516108d6435705ba0 +R 30abdb0021008a8a1927382f64246cc3 U drh -Z 0b33966f147041d5e60fb39f3e8d2b14 +Z a20715ac16e756fff0bd1afe1caf81e7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7e63185ea7..2296089167 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98b0e830bc7effa3c9cbb77aebe7c128afc3b210af336b7516108d6435705ba0 \ No newline at end of file +db8230e80ed894fc2801cdbb53c2cd9de00192d7f4d8328c29cc214e2e0adde9 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 44744524ab..32d655a751 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6122,8 +6122,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ int iAgg = pExpr->iAgg; Parse *pParse = pWalker->pParse; sqlite3 *db = pParse->db; - assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_AGG_FUNCTION ); - if( pExpr->op==TK_AGG_COLUMN ){ + if( pExpr->op!=TK_AGG_FUNCTION ){ + assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_IF_NULL_ROW ); assert( iAgg>=0 && iAggnColumn ); if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); @@ -6133,6 +6133,7 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ } } }else{ + assert( pExpr->op==TK_AGG_FUNCTION ); assert( iAgg>=0 && iAggnFunc ); if( pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); From 3c8fb6fb8461cb813b3b76048ce54e3c1604a06e Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 26 Jul 2022 01:20:18 +0000 Subject: [PATCH 103/151] Performance optimization: avoid unnecessary calls to computeLimitRegisters(). FossilOrigin-Name: f48bd8f85d86fd93329dda7ba57f468854a732b89e4126e9c5beec3495d24b31 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index af92ee66bd..7173617ca0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\sthat\swas\smade\sout-of-date\sby\s[b52393ac28debe98]. -D 2022-07-25T23:34:14.245 +C Performance\soptimization:\s\savoid\sunnecessary\scalls\sto\scomputeLimitRegisters(). +D 2022-07-26T01:20:18.198 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -570,7 +570,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c a9516e1453241986f1eb73c00a0e7cf23d23081fd2eb50e67fcbb96fe7bf6f00 +F src/select.c 2219591ff134d612e2e3e9d401665634bac45956f2c12100280c04efb0d99bb8 F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 F src/sqlite.h.in be265991edca9aea69986758b58ba81cbf5ae403fe0c4ea1d0c9df0cdc8f25ed F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 98b0e830bc7effa3c9cbb77aebe7c128afc3b210af336b7516108d6435705ba0 -R 30abdb0021008a8a1927382f64246cc3 +P db8230e80ed894fc2801cdbb53c2cd9de00192d7f4d8328c29cc214e2e0adde9 +R ff9bb5a29a45c7eaa1b390df1d36ecd7 U drh -Z a20715ac16e756fff0bd1afe1caf81e7 +Z 3b35c17a3beaeddc52e4b88d9f02979e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2296089167..65ce96a9c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db8230e80ed894fc2801cdbb53c2cd9de00192d7f4d8328c29cc214e2e0adde9 \ No newline at end of file +f48bd8f85d86fd93329dda7ba57f468854a732b89e4126e9c5beec3495d24b31 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 672461d447..fe30e96ce0 100644 --- a/src/select.c +++ b/src/select.c @@ -7196,7 +7196,7 @@ int sqlite3Select( if( (p->selFlags & SF_FixedLimit)==0 ){ p->nSelectRow = 320; /* 4 billion rows */ } - computeLimitRegisters(pParse, p, iEnd); + if( p->pLimit ) computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); sSort.sortFlags |= SORTFLAG_UseSorter; From e6710e83fcec5678f059aec7dcd2e20aacfacbdd Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 26 Jul 2022 10:16:25 +0000 Subject: [PATCH 104/151] Add a OOM term to a single assert() statement. FossilOrigin-Name: 217fc3ebd423e050508d6af97e0664f301824ef9a997ed2d302c006a41227917 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7173617ca0..4f61cf9bfc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization:\s\savoid\sunnecessary\scalls\sto\scomputeLimitRegisters(). -D 2022-07-26T01:20:18.198 +C Add\sa\sOOM\sterm\sto\sa\ssingle\sassert()\sstatement. +D 2022-07-26T10:16:25.603 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/vdbe.c 1266f3a4744224253dd74f0080014be8056b062c6f2f6a81e229fa0d306d4102 F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c d68267db6e6641994e17c70670c40fd67ceb2352e42188815ed8c05d4d6502cb -F src/vdbeaux.c f14f30892ee8491f7903cf4ccd7bd0b7ab6f1eddf7dda4a0e03725fcd5ae05e1 +F src/vdbeaux.c 8584f4a20997fd918f0d957ab4f73d7411159772297c5020c188973bdc41dbf0 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 5ebf05c0182addedb1607ade848e1c83cef40981df94d1abfab0c59288c6064f F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db8230e80ed894fc2801cdbb53c2cd9de00192d7f4d8328c29cc214e2e0adde9 -R ff9bb5a29a45c7eaa1b390df1d36ecd7 +P f48bd8f85d86fd93329dda7ba57f468854a732b89e4126e9c5beec3495d24b31 +R e7b7ace3206c3ff792140e3157349b46 U drh -Z 3b35c17a3beaeddc52e4b88d9f02979e +Z f1efcdf9820c3825f4f57a8fb233e014 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 65ce96a9c4..28819b56af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f48bd8f85d86fd93329dda7ba57f468854a732b89e4126e9c5beec3495d24b31 \ No newline at end of file +217fc3ebd423e050508d6af97e0664f301824ef9a997ed2d302c006a41227917 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index dfee43d05c..8ab7978fa1 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1141,7 +1141,7 @@ void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ sqlite3VdbeGetOp(p,addr)->p1 = val; } void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ - assert( addr>=0 ); + assert( addr>=0 || p->db->mallocFailed ); sqlite3VdbeGetOp(p,addr)->p2 = val; } void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ From f4c291275b6cb345afad5feb4f8ecb88ab356e4c Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 26 Jul 2022 14:37:04 +0000 Subject: [PATCH 105/151] Improved AggInfo.aCol debugging output, intended to debug a problem with LEFT JOIN flattening into an aggregate query with GROUP BY. FossilOrigin-Name: e717e029bde4ee68b6ea77a68721c02ddb6e296f1d310a368137ea3c4164f68c --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/expr.c | 2 +- src/select.c | 9 +++++++-- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4f61cf9bfc..c3d2195875 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sOOM\sterm\sto\sa\ssingle\sassert()\sstatement. -D 2022-07-26T10:16:25.603 +C Improved\sAggInfo.aCol\sdebugging\soutput,\sintended\sto\sdebug\sa\sproblem\swith\nLEFT\sJOIN\sflattening\sinto\san\saggregate\squery\swith\sGROUP\sBY. +D 2022-07-26T14:37:04.119 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 13eca2beee5b758ed033a11230971310cc4a58fcd8f6bc33cad4f677c985e96c -F src/expr.c 1565349664995e930cc11217cc665875d8d3d3cae62c92805a67ab582befc92a +F src/expr.c ed5f44ba634193eaa9c603da52ab45a3acd3f23abf786ea0aff69a64da52d20f F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -570,7 +570,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 2219591ff134d612e2e3e9d401665634bac45956f2c12100280c04efb0d99bb8 +F src/select.c 7d385fbf6ba7f386f24d31ba6c508613131680ff4244b75046278a7ac85819c6 F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 F src/sqlite.h.in be265991edca9aea69986758b58ba81cbf5ae403fe0c4ea1d0c9df0cdc8f25ed F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1981,8 +1981,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f48bd8f85d86fd93329dda7ba57f468854a732b89e4126e9c5beec3495d24b31 -R e7b7ace3206c3ff792140e3157349b46 +P 217fc3ebd423e050508d6af97e0664f301824ef9a997ed2d302c006a41227917 +R 05b53d25dc1ad05451162b3588f8e8f6 +T *branch * flatten-left-join +T *sym-flatten-left-join * +T -sym-trunk * U drh -Z f1efcdf9820c3825f4f57a8fb233e014 +Z 41b04475f9d85a45d0a38cd9eef9070b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 28819b56af..d7c0db8b97 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -217fc3ebd423e050508d6af97e0664f301824ef9a997ed2d302c006a41227917 \ No newline at end of file +e717e029bde4ee68b6ea77a68721c02ddb6e296f1d310a368137ea3c4164f68c \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 32d655a751..5f2068d26a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4088,7 +4088,7 @@ expr_code_doover: pCol->iSorterColumn, target); if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else{ + }else if( pTab!=0 ){ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ diff --git a/src/select.c b/src/select.c index fe30e96ce0..d9e0e87698 100644 --- a/src/select.c +++ b/src/select.c @@ -7418,8 +7418,13 @@ int sqlite3Select( sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); } for(ii=0; iinColumn; ii++){ - sqlite3DebugPrintf("agg-column[%d] iMem=%d\n", - ii, pAggInfo->aCol[ii].iMem); + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, pCol->iMem, + pCol->iSorterColumn); sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); } for(ii=0; iinFunc; ii++){ From 4b1b65ca2e561c11adc9466849faea58be7221aa Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 26 Jul 2022 15:32:02 +0000 Subject: [PATCH 106/151] Make sure IF_NULL_ROW expressions receive a separate slot in the sorter used to implement GROUP BY. FossilOrigin-Name: 2bda4fca06ab6be5ad02377a7d1fd9fb9586e3181f1052e4b4937958bdd45efe --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/expr.c | 16 ++++++++++------ src/select.c | 1 + 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c3d2195875..2df75e5e74 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sAggInfo.aCol\sdebugging\soutput,\sintended\sto\sdebug\sa\sproblem\swith\nLEFT\sJOIN\sflattening\sinto\san\saggregate\squery\swith\sGROUP\sBY. -D 2022-07-26T14:37:04.119 +C Make\ssure\sIF_NULL_ROW\sexpressions\sreceive\sa\sseparate\sslot\sin\sthe\ssorter\sused\nto\simplement\sGROUP\sBY. +D 2022-07-26T15:32:02.206 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -521,7 +521,7 @@ F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 13eca2beee5b758ed033a11230971310cc4a58fcd8f6bc33cad4f677c985e96c -F src/expr.c ed5f44ba634193eaa9c603da52ab45a3acd3f23abf786ea0aff69a64da52d20f +F src/expr.c 0f72468b64eef40c280fd6b273c19f3b221444b00f535c7ba6f34faa657e63da F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -570,7 +570,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 7d385fbf6ba7f386f24d31ba6c508613131680ff4244b75046278a7ac85819c6 +F src/select.c 76708e3a8cb80f011f31f1eb6fa5f433aef8f47c85c0a2709148d673a719a846 F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 F src/sqlite.h.in be265991edca9aea69986758b58ba81cbf5ae403fe0c4ea1d0c9df0cdc8f25ed F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1981,11 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 217fc3ebd423e050508d6af97e0664f301824ef9a997ed2d302c006a41227917 -R 05b53d25dc1ad05451162b3588f8e8f6 -T *branch * flatten-left-join -T *sym-flatten-left-join * -T -sym-trunk * +P e717e029bde4ee68b6ea77a68721c02ddb6e296f1d310a368137ea3c4164f68c +R 1029f8a12febed8db153949d4d164b76 U drh -Z 41b04475f9d85a45d0a38cd9eef9070b +Z b08c39293252863d670f80901346b8c8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d7c0db8b97..fb3f6bc93b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e717e029bde4ee68b6ea77a68721c02ddb6e296f1d310a368137ea3c4164f68c \ No newline at end of file +2bda4fca06ab6be5ad02377a7d1fd9fb9586e3181f1052e4b4937958bdd45efe \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 5f2068d26a..be3e73294f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4088,7 +4088,7 @@ expr_code_doover: pCol->iSorterColumn, target); if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); - }else if( pTab!=0 ){ + }else if( ALWAYS(pTab!=0) ){ VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ @@ -6227,8 +6227,10 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ int k; pCol = pAggInfo->aCol; for(k=0; knColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable && - pCol->iColumn==pExpr->iColumn ){ + if( pCol->iTable==pExpr->iTable + && pCol->iColumn==pExpr->iColumn + && pExpr->op!=TK_IF_NULL_ROW + ){ break; } } @@ -6243,15 +6245,17 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ pCol->iMem = ++pParse->nMem; pCol->iSorterColumn = -1; pCol->pCExpr = pExpr; - if( pAggInfo->pGroupBy ){ + if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ int j, n; ExprList *pGB = pAggInfo->pGroupBy; struct ExprList_item *pTerm = pGB->a; n = pGB->nExpr; for(j=0; jpExpr; - if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable && - pE->iColumn==pExpr->iColumn ){ + if( pE->op==TK_COLUMN + && pE->iTable==pExpr->iTable + && pE->iColumn==pExpr->iColumn + ){ pCol->iSorterColumn = j; break; } diff --git a/src/select.c b/src/select.c index d9e0e87698..d5c65bfb28 100644 --- a/src/select.c +++ b/src/select.c @@ -3798,6 +3798,7 @@ static Expr *substExpr( ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; ifNullRow.iTable = pSubst->iNewTable; + ifNullRow.iColumn = -99; ifNullRow.flags = EP_IfNullRow; pCopy = &ifNullRow; } From 4784a78d1cc87525c03e9788307acd0e93344261 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Jul 2022 15:39:32 +0000 Subject: [PATCH 107/151] Add test cases related to fix [e717e029]. FossilOrigin-Name: 2909f36bfba8f34b818e5d9ecd8c759f05bbe25765089e78bb022de2b77a4f0f --- manifest | 14 +++++------ manifest.uuid | 2 +- test/select3.test | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2df75e5e74..eeccada10e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sIF_NULL_ROW\sexpressions\sreceive\sa\sseparate\sslot\sin\sthe\ssorter\sused\nto\simplement\sGROUP\sBY. -D 2022-07-26T15:32:02.206 +C Add\stest\scases\srelated\sto\sfix\s[e717e029]. +D 2022-07-26T15:39:32.378 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1391,7 +1391,7 @@ F test/securedel.test 2f70b2449186a1921bd01ec9da407fbfa98c3a7a5521854c300c194b2f F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/select1.test 692e84cfa29c405854c69e8a4027183d64c22952866a123fabbce741a379e889 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 -F test/select3.test 054b155a4b9394c6858640029cb93e87defbaecc1c87ebb21157c3d35dfc4d88 +F test/select3.test 8d04b66df7475275a65f7e4a786d6a724c30bd9929f8ae5bd59c8d3d6e75e6cd F test/select4.test f0684d3da3bccacbe2a1ebadf6fb49d9df6f53acb4c6ebc228a88d0d6054cc7b F test/select5.test 8afc5e5dcdebc2be54472e73ebd9cd1adef1225fd15d37a1c62f969159f390ae F test/select6.test 9b2fb4ffedf52e1b5703cfcae1212e7a4a063f014c0458d78d29aca3db766d1f @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e717e029bde4ee68b6ea77a68721c02ddb6e296f1d310a368137ea3c4164f68c -R 1029f8a12febed8db153949d4d164b76 -U drh -Z b08c39293252863d670f80901346b8c8 +P 2bda4fca06ab6be5ad02377a7d1fd9fb9586e3181f1052e4b4937958bdd45efe +R 12b2c22b8ec2610d9abb28e0edce627b +U dan +Z 31e4b968c08d2950ff44e5d5f67d5a40 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fb3f6bc93b..b7c6c8eb83 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2bda4fca06ab6be5ad02377a7d1fd9fb9586e3181f1052e4b4937958bdd45efe \ No newline at end of file +2909f36bfba8f34b818e5d9ecd8c759f05bbe25765089e78bb022de2b77a4f0f \ No newline at end of file diff --git a/test/select3.test b/test/select3.test index ec0ee8d2bb..4c9d71b4f5 100644 --- a/test/select3.test +++ b/test/select3.test @@ -374,5 +374,64 @@ foreach {tn sql} { } {abc constant} } +reset_db +do_execsql_test 12.0 { + CREATE TABLE t1(a); + CREATE TABLE t2(x); +} +do_execsql_test 12.1 { + SELECT count(x), m FROM t1 LEFT JOIN (SELECT x, 59 AS m FROM t2) GROUP BY a; +} +do_execsql_test 12.2 { + INSERT INTO t1 VALUES(1), (1), (2), (3); + SELECT count(x), m FROM t1 LEFT JOIN (SELECT x, 59 AS m FROM t2) GROUP BY a; +} { + 0 {} + 0 {} + 0 {} +} +do_execsql_test 12.3 { + INSERT INTO t2 VALUES(45); + SELECT count(x), m FROM t1 LEFT JOIN (SELECT x, 59 AS m FROM t2) GROUP BY a; +} { + 2 59 + 1 59 + 1 59 +} +do_execsql_test 12.4 { + INSERT INTO t2 VALUES(210); + SELECT count(x), m FROM t1 LEFT JOIN (SELECT x, 59 AS m FROM t2) GROUP BY a; +} { + 4 59 + 2 59 + 2 59 +} +do_execsql_test 12.5 { + INSERT INTO t2 VALUES(NULL); + SELECT count(x), m FROM t1 LEFT JOIN (SELECT x, 59 AS m FROM t2) GROUP BY a; +} { + 4 59 + 2 59 + 2 59 +} +do_execsql_test 12.6 { + DELETE FROM t2; + DELETE FROM t1; + INSERT INTO t1 VALUES('value'); + INSERT INTO t2 VALUES('hello'); +} {} +do_execsql_test 12.7 { + SELECT group_concat(x), m FROM t1 + LEFT JOIN (SELECT x, 59 AS m FROM t2) GROUP BY a; +} { + hello 59 +} +do_execsql_test 12.8 { + SELECT group_concat(x), m, n FROM t1 + LEFT JOIN (SELECT x, 59 AS m, 60 AS n FROM t2) GROUP BY a; +} { + hello 59 60 +} finish_test + From 1dd130360020fa9a1cea9f9fb9349cd695df73c5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 26 Jul 2022 15:41:34 +0000 Subject: [PATCH 108/151] Omit a unused variable initialization in order to suppress a harmless compiler warning. FossilOrigin-Name: 2d703c5e941cac72efc21d58e1e35c446b338b2927a42f587ab788826fd7f3f1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4f61cf9bfc..10030b8131 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sOOM\sterm\sto\sa\ssingle\sassert()\sstatement. -D 2022-07-26T10:16:25.603 +C Omit\sa\sunused\svariable\sinitialization\sin\sorder\sto\ssuppress\sa\sharmless\scompiler\nwarning. +D 2022-07-26T15:41:34.792 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 1307d57f65023c9f37c2b6c62253343fca63877f161f694a593b14c0883cedda +F src/btree.c 4a8d349b9ed4dc6d252c535227699d75319b633058a56432ebf43c9f56f9085e F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e F src/build.c 33aec2b34dbc53c3da2ea8434fccbf3c17c35ba647c3193df598cce0e861355c @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f48bd8f85d86fd93329dda7ba57f468854a732b89e4126e9c5beec3495d24b31 -R e7b7ace3206c3ff792140e3157349b46 +P 217fc3ebd423e050508d6af97e0664f301824ef9a997ed2d302c006a41227917 +R 407e9b273b183df6bef313b5323c800a U drh -Z f1efcdf9820c3825f4f57a8fb233e014 +Z a0cbd446115797b84cb35ad185602ef9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 28819b56af..814b7ff643 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -217fc3ebd423e050508d6af97e0664f301824ef9a997ed2d302c006a41227917 \ No newline at end of file +2d703c5e941cac72efc21d58e1e35c446b338b2927a42f587ab788826fd7f3f1 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 80bca66353..09835fca03 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1513,7 +1513,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - src = data = pPage->aData; + data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; From 3b01dd0f5db44d5d236326cb948a0cac048d5024 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 26 Jul 2022 19:10:13 +0000 Subject: [PATCH 109/151] Restore a VDBE coverage macro that is mistakenly deleted for check-in [92ac01d41d46ab73]. FossilOrigin-Name: 01de7ec44fb1e8e6f847eaef433216b3163a44a8789451718e1c419ef840ddfb --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/window.c | 1 + 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6408bbbc2e..3281ad26af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sproblem\srelated\sto\saggregate\squery\sLEFT\sJOIN\sflattening. -D 2022-07-26T18:43:45.800 +C Restore\sa\sVDBE\scoverage\smacro\sthat\sis\smistakenly\sdeleted\sfor\ncheck-in\s[92ac01d41d46ab73]. +D 2022-07-26T19:10:13.436 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -661,7 +661,7 @@ F src/where.c 1049685e84bd74692ad76984a3411a21c5a1e6ddd08c981ec94d2f11f769e07f F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 F src/wherecode.c 210240c9cec2a1d1494a3ac7852e9432245382e752b490c2bb437637c2c98711 F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 -F src/window.c 057df98b3b8296c4ccd35066ee171b66d956b3ee8bfce27596ddb5ed8fab66f4 +F src/window.c 928e215840e2f2d9a2746e018c9643ef42c66c4ab6630ef0df7fa388fa145e86 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test b5c19d504dec222c0dc66642673d23dce915d35737b68e74d9f237b80493eb53 @@ -1981,9 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2d703c5e941cac72efc21d58e1e35c446b338b2927a42f587ab788826fd7f3f1 2909f36bfba8f34b818e5d9ecd8c759f05bbe25765089e78bb022de2b77a4f0f -R fd8805f95598ecc80e3e7fdb82e7f06b -T +closed 2909f36bfba8f34b818e5d9ecd8c759f05bbe25765089e78bb022de2b77a4f0f -U dan -Z 98cbf6cf155acbce70008bbc87cc3bcc +P 8e12c6f59bcb67be4ff131feecc5af68f4bb0afbcced2e2f7e73289aaf95d99d +R d0d3075ca4c78be3d08a2ae674010fb3 +U drh +Z 2ec897bd518817fd9fabfa62106fc627 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8ac3fc2520..3b870e435e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e12c6f59bcb67be4ff131feecc5af68f4bb0afbcced2e2f7e73289aaf95d99d \ No newline at end of file +01de7ec44fb1e8e6f847eaef433216b3163a44a8789451718e1c419ef840ddfb \ No newline at end of file diff --git a/src/window.c b/src/window.c index 030365851c..f13ea8b027 100644 --- a/src/window.c +++ b/src/window.c @@ -2172,6 +2172,7 @@ static void windowCodeRangeTest( sqlite3VdbeJumpHere(v, addr); sqlite3VdbeAddOp2(v, OP_IsNull, reg2, (op==OP_Gt || op==OP_Ge) ? addrDone : lbl); + VdbeCoverage(v); } /* Register reg1 currently contains csr1.peerVal (the peer-value from csr1). From f98109e7cb51e0950ed8fe863f970ea6f9e729bd Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 1 Aug 2022 13:14:03 +0000 Subject: [PATCH 110/151] In the xUpdate method of the GeoPoly virtual table, make sure that the number of updated columns does not exceed the underlying implementation, even if the virtual table object records an excess number of column in the nAux field due to table constraints in the table definition. Fix for the problem reported by [forum:/forumpost/a096ab7d96bb057a|forum post a096ab7d96bb057a]. FossilOrigin-Name: a04dd475c2c8afdded0afecfd34c6c4c2f48cce520e09b7fa5172ff30f09209e --- ext/rtree/geopoly.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/rtree/geopoly.c b/ext/rtree/geopoly.c index 68cde87344..7b41e79092 100644 --- a/ext/rtree/geopoly.c +++ b/ext/rtree/geopoly.c @@ -1695,7 +1695,7 @@ static int geopolyUpdate( sqlite3_free(p); nChange = 1; } - for(jj=1; jjnAux; jj++){ + for(jj=1; jj Date: Wed, 3 Aug 2022 15:58:44 +0000 Subject: [PATCH 111/151] For an IN operator used with a RIGHT JOIN, use the number of terms in the vector, not the number of equality terms, to size the column map. dbsqlfuzz 14cfdad6ca45e607163f54049ddf5065183dc657. FossilOrigin-Name: b184c8d9222da6b420b5d7c439bbe4b737d92ffa27f5e933f4e1a9c8117b0ee5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wherecode.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c3727e9837..f3994472d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sxUpdate\smethod\sof\sthe\sGeoPoly\svirtual\stable,\smake\ssure\sthat\sthe\nnumber\sof\supdated\scolumns\sdoes\snot\sexceed\sthe\sunderlying\simplementation,\seven\nif\sthe\svirtual\stable\sobject\srecords\san\sexcess\snumber\sof\scolumn\sin\sthe\snAux\nfield\sdue\sto\stable\sconstraints\sin\sthe\stable\sdefinition.\nFix\sfor\sthe\sproblem\sreported\sby\n[forum:/forumpost/a096ab7d96bb057a|forum\spost\sa096ab7d96bb057a]. -D 2022-08-01T13:14:03.301 +C For\san\sIN\soperator\sused\swith\sa\sRIGHT\sJOIN,\suse\sthe\snumber\sof\sterms\sin\sthe\nvector,\snot\sthe\snumber\sof\sequality\sterms,\sto\ssize\sthe\scolumn\smap.\ndbsqlfuzz\s14cfdad6ca45e607163f54049ddf5065183dc657. +D 2022-08-03T15:58:44.012 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -659,7 +659,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 1049685e84bd74692ad76984a3411a21c5a1e6ddd08c981ec94d2f11f769e07f F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 -F src/wherecode.c 210240c9cec2a1d1494a3ac7852e9432245382e752b490c2bb437637c2c98711 +F src/wherecode.c 6fc48e5b25821c66c387a2e5b2d00bf497636ed330683ce5acae50b178d25bac F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 F src/window.c 928e215840e2f2d9a2746e018c9643ef42c66c4ab6630ef0df7fa388fa145e86 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 01de7ec44fb1e8e6f847eaef433216b3163a44a8789451718e1c419ef840ddfb -R 6e9af4a1fd48ba4a0b2f74feb2fb7ec8 +P a04dd475c2c8afdded0afecfd34c6c4c2f48cce520e09b7fa5172ff30f09209e +R 63dd5d94fb138ca07714f601c94938cf U drh -Z b4532202f959fae679f2c4c7df6a9a48 +Z 4f99307534c0bc9f7c784d197ef60b08 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index db1cc4977c..d44341365c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a04dd475c2c8afdded0afecfd34c6c4c2f48cce520e09b7fa5172ff30f09209e \ No newline at end of file +b184c8d9222da6b420b5d7c439bbe4b737d92ffa27f5e933f4e1a9c8117b0ee5 \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index db7c60ec2a..ab947773b7 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -622,7 +622,8 @@ static int codeEqualityTerm( } sqlite3ExprDelete(db, pX); }else{ - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, + sizeof(int)*sqlite3ExprVectorSize(pX->pLeft)); eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); } pX = pExpr; From 3a17e2e5cf6896486ecdda92ef6abc7a9e74ec6c Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 3 Aug 2022 17:26:32 +0000 Subject: [PATCH 112/151] Improvement on the previous check-in. FossilOrigin-Name: c1ab0ea2959259cbd29f624bfd5e7366035f2dbcaab72c0bf55723c0919b7a79 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wherecode.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f3994472d9..e053d1a255 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\san\sIN\soperator\sused\swith\sa\sRIGHT\sJOIN,\suse\sthe\snumber\sof\sterms\sin\sthe\nvector,\snot\sthe\snumber\sof\sequality\sterms,\sto\ssize\sthe\scolumn\smap.\ndbsqlfuzz\s14cfdad6ca45e607163f54049ddf5065183dc657. -D 2022-08-03T15:58:44.012 +C Improvement\son\sthe\sprevious\scheck-in. +D 2022-08-03T17:26:32.426 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -659,7 +659,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 1049685e84bd74692ad76984a3411a21c5a1e6ddd08c981ec94d2f11f769e07f F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 -F src/wherecode.c 6fc48e5b25821c66c387a2e5b2d00bf497636ed330683ce5acae50b178d25bac +F src/wherecode.c 719a5bb0102711a6ebed8d4385b831a27ce679172f3ae8e9aaec0dc1e415a95a F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 F src/window.c 928e215840e2f2d9a2746e018c9643ef42c66c4ab6630ef0df7fa388fa145e86 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a04dd475c2c8afdded0afecfd34c6c4c2f48cce520e09b7fa5172ff30f09209e -R 63dd5d94fb138ca07714f601c94938cf +P b184c8d9222da6b420b5d7c439bbe4b737d92ffa27f5e933f4e1a9c8117b0ee5 +R 600a480393721dba1f30f4d7f2bbbe2b U drh -Z 4f99307534c0bc9f7c784d197ef60b08 +Z c1a9cd59eb2cac4d212215eae9db9a45 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d44341365c..1e485a198c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b184c8d9222da6b420b5d7c439bbe4b737d92ffa27f5e933f4e1a9c8117b0ee5 \ No newline at end of file +c1ab0ea2959259cbd29f624bfd5e7366035f2dbcaab72c0bf55723c0919b7a79 \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index ab947773b7..85df64279c 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -622,8 +622,8 @@ static int codeEqualityTerm( } sqlite3ExprDelete(db, pX); }else{ - aiMap = (int*)sqlite3DbMallocZero(pParse->db, - sizeof(int)*sqlite3ExprVectorSize(pX->pLeft)); + int n = sqlite3ExprVectorSize(pX->pLeft); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); } pX = pExpr; From 403f00219b78fe8778af1f487c04d840a3abe0f5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 3 Aug 2022 19:37:25 +0000 Subject: [PATCH 113/151] Small performance increase on the binding interfaces. FossilOrigin-Name: aab24c37fb444804fb91177b4b522909ef1bb85383444d97d82a758f919047d2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 17 ++++++++--------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index e053d1a255..7263c92943 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvement\son\sthe\sprevious\scheck-in. -D 2022-08-03T17:26:32.426 +C Small\sperformance\sincrease\son\sthe\sbinding\sinterfaces. +D 2022-08-03T19:37:25.175 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -645,7 +645,7 @@ F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 F src/vdbe.c 1266f3a4744224253dd74f0080014be8056b062c6f2f6a81e229fa0d306d4102 F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da -F src/vdbeapi.c d68267db6e6641994e17c70670c40fd67ceb2352e42188815ed8c05d4d6502cb +F src/vdbeapi.c 4181dffeef6ee181275cfecf27ad8577024c17ee9639531decdf4162b8c0326b F src/vdbeaux.c 8584f4a20997fd918f0d957ab4f73d7411159772297c5020c188973bdc41dbf0 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 5ebf05c0182addedb1607ade848e1c83cef40981df94d1abfab0c59288c6064f @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b184c8d9222da6b420b5d7c439bbe4b737d92ffa27f5e933f4e1a9c8117b0ee5 -R 600a480393721dba1f30f4d7f2bbbe2b +P c1ab0ea2959259cbd29f624bfd5e7366035f2dbcaab72c0bf55723c0919b7a79 +R 6d30ee627cafae056fe4256608756484 U drh -Z c1a9cd59eb2cac4d212215eae9db9a45 +Z 20186bacd0ffc8dda8aa3c7b92c68f3f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1e485a198c..0fa5666afd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1ab0ea2959259cbd29f624bfd5e7366035f2dbcaab72c0bf55723c0919b7a79 \ No newline at end of file +aab24c37fb444804fb91177b4b522909ef1bb85383444d97d82a758f919047d2 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 97bc744c30..c810610bef 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1432,7 +1432,7 @@ const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ ** The error code stored in database p->db is overwritten with the return ** value in any case. */ -static int vdbeUnbind(Vdbe *p, int i){ +static int vdbeUnbind(Vdbe *p, unsigned int i){ Mem *pVar; if( vdbeSafetyNotNull(p) ){ return SQLITE_MISUSE_BKPT; @@ -1445,12 +1445,11 @@ static int vdbeUnbind(Vdbe *p, int i){ "bind on a busy prepared statement: [%s]", p->zSql); return SQLITE_MISUSE_BKPT; } - if( i<1 || i>p->nVar ){ + if( i>=p->nVar ){ sqlite3Error(p->db, SQLITE_RANGE); sqlite3_mutex_leave(p->db->mutex); return SQLITE_RANGE; } - i--; pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; @@ -1487,7 +1486,7 @@ static int bindText( Mem *pVar; int rc; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ if( zData!=0 ){ pVar = &p->aVar[i-1]; @@ -1536,7 +1535,7 @@ int sqlite3_bind_blob64( int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); sqlite3_mutex_leave(p->db->mutex); @@ -1549,7 +1548,7 @@ int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); sqlite3_mutex_leave(p->db->mutex); @@ -1559,7 +1558,7 @@ int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ int rc; Vdbe *p = (Vdbe*)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3_mutex_leave(p->db->mutex); } @@ -1574,7 +1573,7 @@ int sqlite3_bind_pointer( ){ int rc; Vdbe *p = (Vdbe*)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor); sqlite3_mutex_leave(p->db->mutex); @@ -1652,7 +1651,7 @@ int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_INCRBLOB sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); From 3cf46ee508e97b46736a2607ded9c84c2c16229f Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 3 Aug 2022 19:53:54 +0000 Subject: [PATCH 114/151] Performance optimization to the math opcodes of the bytecode engine. FossilOrigin-Name: 597347d9c39f6ed068ac942a8c9853b1b745fce1eb5e9dd30f6c1cb312b36f53 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 12 +++++++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 7263c92943..75438ca23f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\sincrease\son\sthe\sbinding\sinterfaces. -D 2022-08-03T19:37:25.175 +C Performance\soptimization\sto\sthe\smath\sopcodes\sof\sthe\sbytecode\sengine. +D 2022-08-03T19:53:54.082 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 602fe229f32a96ceccae4f40824129669582096f7c355f53dbac156c9fecef23 F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 -F src/vdbe.c 1266f3a4744224253dd74f0080014be8056b062c6f2f6a81e229fa0d306d4102 +F src/vdbe.c 34f76cbbd886d90413b4503e6d46d7425152b88219bdd4e6bcaf81da75640d3e F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c 4181dffeef6ee181275cfecf27ad8577024c17ee9639531decdf4162b8c0326b @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c1ab0ea2959259cbd29f624bfd5e7366035f2dbcaab72c0bf55723c0919b7a79 -R 6d30ee627cafae056fe4256608756484 +P aab24c37fb444804fb91177b4b522909ef1bb85383444d97d82a758f919047d2 +R e2ed79990c9f576213c6203a5349f2b7 U drh -Z 20186bacd0ffc8dda8aa3c7b92c68f3f +Z ef554432af87d216083ec59e16ec0e61 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0fa5666afd..a35147d8e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aab24c37fb444804fb91177b4b522909ef1bb85383444d97d82a758f919047d2 \ No newline at end of file +597347d9c39f6ed068ac942a8c9853b1b745fce1eb5e9dd30f6c1cb312b36f53 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 5a6a8fbdc2..0b6fee3521 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1731,7 +1731,6 @@ case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ - u16 flags; /* Combined MEM_* flags from both inputs */ u16 type1; /* Numeric type of left operand */ u16 type2; /* Numeric type of right operand */ i64 iA; /* Integer value of left operand */ @@ -1740,12 +1739,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ double rB; /* Real value of right operand */ pIn1 = &aMem[pOp->p1]; - type1 = numericType(pIn1); + type1 = pIn1->flags; pIn2 = &aMem[pOp->p2]; - type2 = numericType(pIn2); + type2 = pIn2->flags; pOut = &aMem[pOp->p3]; - flags = pIn1->flags | pIn2->flags; if( (type1 & type2 & MEM_Int)!=0 ){ +int_math: iA = pIn1->u.i; iB = pIn2->u.i; switch( pOp->opcode ){ @@ -1767,9 +1766,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ } pOut->u.i = iB; MemSetTypeFlag(pOut, MEM_Int); - }else if( (flags & MEM_Null)!=0 ){ + }else if( ((type1 | type2) & MEM_Null)!=0 ){ goto arithmetic_result_is_null; }else{ + type1 = numericType(pIn1); + type2 = numericType(pIn2); + if( (type1 & type2 & MEM_Int)!=0 ) goto int_math; fp_math: rA = sqlite3VdbeRealValue(pIn1); rB = sqlite3VdbeRealValue(pIn2); From e24a6f58ae24670446f386578e63962633e8d4a6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 4 Aug 2022 14:02:35 +0000 Subject: [PATCH 115/151] Fix a typo on a comment used to generate bytecode documentation. FossilOrigin-Name: 64348d0b665f61c3a89f51341f23ca87f939dd9e306dc58c2004e46ab04e696e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 75438ca23f..3e7707eefe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sto\sthe\smath\sopcodes\sof\sthe\sbytecode\sengine. -D 2022-08-03T19:53:54.082 +C Fix\sa\stypo\son\sa\scomment\sused\sto\sgenerate\sbytecode\sdocumentation. +D 2022-08-04T14:02:35.852 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 602fe229f32a96ceccae4f40824129669582096f7c355f53dbac156c9fecef23 F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 -F src/vdbe.c 34f76cbbd886d90413b4503e6d46d7425152b88219bdd4e6bcaf81da75640d3e +F src/vdbe.c 796fbae3e989744ea3761c5ab8a23bec1764ff889d267d71666fcaa4aeb75a81 F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c 4181dffeef6ee181275cfecf27ad8577024c17ee9639531decdf4162b8c0326b @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aab24c37fb444804fb91177b4b522909ef1bb85383444d97d82a758f919047d2 -R e2ed79990c9f576213c6203a5349f2b7 +P 597347d9c39f6ed068ac942a8c9853b1b745fce1eb5e9dd30f6c1cb312b36f53 +R ce5a27ffc72e7ec12d095a7689107487 U drh -Z ef554432af87d216083ec59e16ec0e61 +Z 549658791a10dec284226014b5aa7e75 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a35147d8e3..f4402b8b33 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -597347d9c39f6ed068ac942a8c9853b1b745fce1eb5e9dd30f6c1cb312b36f53 \ No newline at end of file +64348d0b665f61c3a89f51341f23ca87f939dd9e306dc58c2004e46ab04e696e \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 0b6fee3521..72dae08810 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -7216,7 +7216,7 @@ case OP_IfPos: { /* jump, in1 */ ** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) ** ** This opcode performs a commonly used computation associated with -** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3] +** LIMIT and OFFSET processing. r[P1] holds the limit counter. r[P3] ** holds the offset counter. The opcode computes the combined value ** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2] ** value computed is the total number of rows that will need to be From bffd5c1ece80bcf04512b2965d06bc89aaab7ba8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 4 Aug 2022 17:15:00 +0000 Subject: [PATCH 116/151] Fix a problem with the query optimizer for LIMIT/OFFSET queries when underlying query is a UNION ALL and both arms of the UNION ALL are subqueries with an ORDER BY clause. This bug was reported at [forum:/forumpost/6b5e9188f0657616|forum post 6b5e9188f0657616]. The problem was introduced in 2015 (SQLite version 3.9.0) by check-in [4b631364354068af]. See also ticket [b65cb2c8d91f6685]. FossilOrigin-Name: 6c806f64bbc9e98891bad0868575ee2ec5d0951ceb0c71d3ed417b45d5f27561 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 5 ++++- test/offset1.test | 43 ++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3e7707eefe..86bc727217 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\son\sa\scomment\sused\sto\sgenerate\sbytecode\sdocumentation. -D 2022-08-04T14:02:35.852 +C Fix\sa\sproblem\swith\sthe\squery\soptimizer\sfor\sLIMIT/OFFSET\squeries\swhen\nunderlying\squery\sis\sa\sUNION\sALL\sand\sboth\sarms\sof\sthe\sUNION\sALL\sare\nsubqueries\swith\san\sORDER\sBY\sclause.\s\sThis\sbug\swas\sreported\sat\n[forum:/forumpost/6b5e9188f0657616|forum\spost\s6b5e9188f0657616].\s\sThe\nproblem\swas\sintroduced\sin\s2015\s(SQLite\sversion\s3.9.0)\sby\scheck-in\n[4b631364354068af].\s\sSee\salso\sticket\s[b65cb2c8d91f6685]. +D 2022-08-04T17:15:00.492 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -570,7 +570,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 76708e3a8cb80f011f31f1eb6fa5f433aef8f47c85c0a2709148d673a719a846 +F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 F src/sqlite.h.in be265991edca9aea69986758b58ba81cbf5ae403fe0c4ea1d0c9df0cdc8f25ed F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1286,7 +1286,7 @@ F test/null.test b7ff206a1c60fe01aa2abd33ef9ea83c93727d993ca8a613de86e925c9f2bc6 F test/nulls1.test 7a5e4346ee4285034100b4cd20e6784f16a9d6c927e44ecdf10034086bbee9c9 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823 -F test/offset1.test f06b83657bcf26f9ce805e67450e189e282143b2 +F test/offset1.test 72cca52482cbd5bc687cfa67aa2566c859081b5a353fd2f9da9bbd3914dea1ef F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/optfuzz-db01.c 9f2fa80b8f84ebbf1f2e8b13421a4e0477fe300f6686fbd76cac1d2db66e0fdc F test/optfuzz-db01.txt 21f6bdeadc701cf11528276e2a55c70bfcb846ba42df327f979bd9e7b6ce7041 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 597347d9c39f6ed068ac942a8c9853b1b745fce1eb5e9dd30f6c1cb312b36f53 -R ce5a27ffc72e7ec12d095a7689107487 +P 64348d0b665f61c3a89f51341f23ca87f939dd9e306dc58c2004e46ab04e696e +R ee3ba899316d027809db4ccd10585a40 U drh -Z 549658791a10dec284226014b5aa7e75 +Z 55de9d5c0f343647378b885b0fe37df5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f4402b8b33..70b6140f8b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -64348d0b665f61c3a89f51341f23ca87f939dd9e306dc58c2004e46ab04e696e \ No newline at end of file +6c806f64bbc9e98891bad0868575ee2ec5d0951ceb0c71d3ed417b45d5f27561 \ No newline at end of file diff --git a/src/select.c b/src/select.c index d5c65bfb28..3f80e44ce2 100644 --- a/src/select.c +++ b/src/select.c @@ -1688,7 +1688,7 @@ static void generateSortTail( if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); - codeOffset(v, p->iOffset, addrContinue); + assert( p->iLimit==0 && p->iOffset==0 ); sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); bSeq = 0; }else{ @@ -1696,6 +1696,9 @@ static void generateSortTail( codeOffset(v, p->iOffset, addrContinue); iSortTab = iTab; bSeq = 1; + if( p->iOffset>0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1); + } } for(i=0, iCol=nKey+bSeq-1; i= 100 + ORDER BY salary asc); +} {} +do_execsql_test offset1-2.1 { + SELECT * FROM v LIMIT 5 OFFSET 2; +} { + 22 Grace Berlin it 90 + 21 Emma London it 84 + 12 Bob London hr 78 + 11 Diane London hr 70 + 33 Alice Berlin sales 100 +} finish_test From e2848932a508e48e8ac7e98a647d63ebc34632f6 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 5 Aug 2022 05:30:07 +0000 Subject: [PATCH 117/151] Fix harmless compiler warning seen with MSVC. FossilOrigin-Name: e36217f2536d89f7a15a879fe3ead32307909c71ced42e1ae7b35f43d4744a80 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeapi.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 86bc727217..16ad722e8f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sthe\squery\soptimizer\sfor\sLIMIT/OFFSET\squeries\swhen\nunderlying\squery\sis\sa\sUNION\sALL\sand\sboth\sarms\sof\sthe\sUNION\sALL\sare\nsubqueries\swith\san\sORDER\sBY\sclause.\s\sThis\sbug\swas\sreported\sat\n[forum:/forumpost/6b5e9188f0657616|forum\spost\s6b5e9188f0657616].\s\sThe\nproblem\swas\sintroduced\sin\s2015\s(SQLite\sversion\s3.9.0)\sby\scheck-in\n[4b631364354068af].\s\sSee\salso\sticket\s[b65cb2c8d91f6685]. -D 2022-08-04T17:15:00.492 +C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC. +D 2022-08-05T05:30:07.478 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -645,7 +645,7 @@ F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 F src/vdbe.c 796fbae3e989744ea3761c5ab8a23bec1764ff889d267d71666fcaa4aeb75a81 F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da -F src/vdbeapi.c 4181dffeef6ee181275cfecf27ad8577024c17ee9639531decdf4162b8c0326b +F src/vdbeapi.c 4cfbf7ec3ed60366a38655f3f10316c5a3d68f6d4d06e462f88679392611c756 F src/vdbeaux.c 8584f4a20997fd918f0d957ab4f73d7411159772297c5020c188973bdc41dbf0 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 5ebf05c0182addedb1607ade848e1c83cef40981df94d1abfab0c59288c6064f @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 64348d0b665f61c3a89f51341f23ca87f939dd9e306dc58c2004e46ab04e696e -R ee3ba899316d027809db4ccd10585a40 -U drh -Z 55de9d5c0f343647378b885b0fe37df5 +P 6c806f64bbc9e98891bad0868575ee2ec5d0951ceb0c71d3ed417b45d5f27561 +R 93de007b6ed89be23aeb731b205e5be2 +U mistachkin +Z 19aad0410826162a6654bad862e7b83b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 70b6140f8b..846e2e7574 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c806f64bbc9e98891bad0868575ee2ec5d0951ceb0c71d3ed417b45d5f27561 \ No newline at end of file +e36217f2536d89f7a15a879fe3ead32307909c71ced42e1ae7b35f43d4744a80 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index c810610bef..5d3e5601e5 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1445,7 +1445,7 @@ static int vdbeUnbind(Vdbe *p, unsigned int i){ "bind on a busy prepared statement: [%s]", p->zSql); return SQLITE_MISUSE_BKPT; } - if( i>=p->nVar ){ + if( i>=(unsigned int)p->nVar ){ sqlite3Error(p->db, SQLITE_RANGE); sqlite3_mutex_leave(p->db->mutex); return SQLITE_RANGE; From 8e658f0a13da40b920241329ec87fe9c9c16b111 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 5 Aug 2022 17:16:00 +0000 Subject: [PATCH 118/151] Avoid having fts3 read uninitialized values when processing deferred tokens. FossilOrigin-Name: 98729c5fa6c4c418ca9779c35e6874c930e5cee5a90b5eb32b284a0f3762269e --- ext/fts3/fts3.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 1a1672512a..3b73fbba1d 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -4376,7 +4376,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nDistance = iPrev - nMaxUndeferred; } - aOut = (char *)sqlite3_malloc(nPoslist+8); + aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING); if( !aOut ){ sqlite3_free(aPoslist); return SQLITE_NOMEM; diff --git a/manifest b/manifest index 16ad722e8f..ab1066930e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC. -D 2022-08-05T05:30:07.478 +C Avoid\shaving\sfts3\sread\suninitialized\svalues\swhen\sprocessing\sdeferred\stokens. +D 2022-08-05T17:16:00.185 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -102,7 +102,7 @@ F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 8cd361bd8612de3e5ec451d33ced8575cbc5af6744e7efc05d54bde4d2df3cd0 +F ext/fts3/fts3.c b20bd583991e740c1e14371896f3ab61d7b6c033740646366bbf6667191bc4e2 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h dafdc371f9fbab175744b06cfe019d5f040cdfdbd11fea752f5dc28d45b04c05 F ext/fts3/fts3_aux.c f0dc9bd98582615b7750218899bd0c729879b6bbf94d1be57ca1833ff49afc6f @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6c806f64bbc9e98891bad0868575ee2ec5d0951ceb0c71d3ed417b45d5f27561 -R 93de007b6ed89be23aeb731b205e5be2 -U mistachkin -Z 19aad0410826162a6654bad862e7b83b +P e36217f2536d89f7a15a879fe3ead32307909c71ced42e1ae7b35f43d4744a80 +R 9810d4cb92a49345fdc5e6b1d4c32ce1 +U dan +Z 17935eeb35e00b5bc2c7ef71b0671232 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 846e2e7574..92d41db7f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e36217f2536d89f7a15a879fe3ead32307909c71ced42e1ae7b35f43d4744a80 \ No newline at end of file +98729c5fa6c4c418ca9779c35e6874c930e5cee5a90b5eb32b284a0f3762269e \ No newline at end of file From 8c3cc71a5ad77541951ec17d88451b227932253c Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 6 Aug 2022 15:28:37 +0000 Subject: [PATCH 119/151] Fix a rounding error caused by scalar->logarithm->scalar conversion when using stat4 data to estimate some range scans. FossilOrigin-Name: 86e30d7c4bb1cebe25b66686d6d1ee170c1a88e5a6c1bed6652ebbb024d62062 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ab1066930e..5cc169bbdb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\shaving\sfts3\sread\suninitialized\svalues\swhen\sprocessing\sdeferred\stokens. -D 2022-08-05T17:16:00.185 +C Fix\sa\srounding\serror\scaused\sby\sscalar->logarithm->scalar\sconversion\swhen\susing\sstat4\sdata\sto\sestimate\ssome\srange\sscans. +D 2022-08-06T15:28:37.948 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -657,7 +657,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 1049685e84bd74692ad76984a3411a21c5a1e6ddd08c981ec94d2f11f769e07f +F src/where.c 8bfe41220558a03be393d064ef3d942cd1ceaf57cd88a85b6cb59b7dda555202 F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 F src/wherecode.c 719a5bb0102711a6ebed8d4385b831a27ce679172f3ae8e9aaec0dc1e415a95a F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e36217f2536d89f7a15a879fe3ead32307909c71ced42e1ae7b35f43d4744a80 -R 9810d4cb92a49345fdc5e6b1d4c32ce1 +P 98729c5fa6c4c418ca9779c35e6874c930e5cee5a90b5eb32b284a0f3762269e +R 33e8f2d20ee794e5cf556ebabd2d9ca9 U dan -Z 17935eeb35e00b5bc2c7ef71b0671232 +Z b5ab7076fb09a8652a3894b7ab774d07 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 92d41db7f5..35773030c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98729c5fa6c4c418ca9779c35e6874c930e5cee5a90b5eb32b284a0f3762269e \ No newline at end of file +86e30d7c4bb1cebe25b66686d6d1ee170c1a88e5a6c1bed6652ebbb024d62062 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6f3402a6e4..9c4678345e 100644 --- a/src/where.c +++ b/src/where.c @@ -1567,7 +1567,7 @@ static int whereKeyStats( ** is larger than all samples in the array. */ tRowcnt iUpper, iGap; if( i>=pIdx->nSample ){ - iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = pIdx->nRowEst0; }else{ iUpper = aSample[i].anLt[iCol]; } From a3a4da09226982a646982f2bc2188ab3c071af25 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 8 Aug 2022 12:19:13 +0000 Subject: [PATCH 120/151] Do not attempt to convert an oversized floating point value into an integer. FossilOrigin-Name: 6c4fc8385ee22516e0b87fb647327ee7d1a564040ebe2a4d66fc999ef2908df2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5cc169bbdb..f5ed72888c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\srounding\serror\scaused\sby\sscalar->logarithm->scalar\sconversion\swhen\susing\sstat4\sdata\sto\sestimate\ssome\srange\sscans. -D 2022-08-06T15:28:37.948 +C Do\snot\sattempt\sto\sconvert\san\soversized\sfloating\spoint\svalue\sinto\san\sinteger. +D 2022-08-08T12:19:13.235 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 602fe229f32a96ceccae4f40824129669582096f7c355f53dbac156c9fecef23 F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 -F src/vdbe.c 796fbae3e989744ea3761c5ab8a23bec1764ff889d267d71666fcaa4aeb75a81 +F src/vdbe.c e0517ef4a599aa71dcc3875c558f0369790b7e048df8ed7b9993d24818a26ea3 F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c 4cfbf7ec3ed60366a38655f3f10316c5a3d68f6d4d06e462f88679392611c756 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 98729c5fa6c4c418ca9779c35e6874c930e5cee5a90b5eb32b284a0f3762269e -R 33e8f2d20ee794e5cf556ebabd2d9ca9 -U dan -Z b5ab7076fb09a8652a3894b7ab774d07 +P 86e30d7c4bb1cebe25b66686d6d1ee170c1a88e5a6c1bed6652ebbb024d62062 +R cc38bc596bc6cfb7cf263b784bfa32d2 +U drh +Z 1b3d5f93d9de8e10c30a689056160d79 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 35773030c3..30ca22856c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -86e30d7c4bb1cebe25b66686d6d1ee170c1a88e5a6c1bed6652ebbb024d62062 \ No newline at end of file +6c4fc8385ee22516e0b87fb647327ee7d1a564040ebe2a4d66fc999ef2908df2 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 72dae08810..346d005fad 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -314,7 +314,9 @@ static VdbeCursor *allocateCursor( ** return false. */ static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ - i64 iValue = (double)rValue; + i64 iValue; + if( rValue>(double)LARGEST_INT64 || rValue<(double)SMALLEST_INT64 ) return 0; + iValue = (double)rValue; if( sqlite3RealSameAsInt(rValue,iValue) ){ *piValue = iValue; return 1; From 5e10d892d699ea0dc48c4fc15d996c9b7629a989 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 8 Aug 2022 13:04:08 +0000 Subject: [PATCH 121/151] Remove an unreachable branch in the numericType() helper routine. FossilOrigin-Name: a99cc008e46ab1a4fdbe2fa87202d026a10c57be55e3e9fedd935795ed6dc694 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 15 ++++++++------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index f5ed72888c..6c2fbdb446 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sattempt\sto\sconvert\san\soversized\sfloating\spoint\svalue\sinto\san\sinteger. -D 2022-08-08T12:19:13.235 +C Remove\san\sunreachable\sbranch\sin\sthe\snumericType()\shelper\sroutine. +D 2022-08-08T13:04:08.230 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 602fe229f32a96ceccae4f40824129669582096f7c355f53dbac156c9fecef23 F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 -F src/vdbe.c e0517ef4a599aa71dcc3875c558f0369790b7e048df8ed7b9993d24818a26ea3 +F src/vdbe.c 6bca2e524d29d9f40e9842c287e60dbae5c428b6ea23f8ceb3114b78b3995d73 F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c 4cfbf7ec3ed60366a38655f3f10316c5a3d68f6d4d06e462f88679392611c756 @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 86e30d7c4bb1cebe25b66686d6d1ee170c1a88e5a6c1bed6652ebbb024d62062 -R cc38bc596bc6cfb7cf263b784bfa32d2 +P 6c4fc8385ee22516e0b87fb647327ee7d1a564040ebe2a4d66fc999ef2908df2 +R e4cfac9ab2cb69853bf392410d0d10ea U drh -Z 1b3d5f93d9de8e10c30a689056160d79 +Z 0989c0895e99c37065974c0506edc50e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 30ca22856c..cbd770bafc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c4fc8385ee22516e0b87fb647327ee7d1a564040ebe2a4d66fc999ef2908df2 \ No newline at end of file +a99cc008e46ab1a4fdbe2fa87202d026a10c57be55e3e9fedd935795ed6dc694 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 346d005fad..dcfab51ace 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -478,17 +478,18 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ ** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ - if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal) ){ + assert( (pMem->flags & MEM_Null)==0 + || pMem->db==0 || pMem->db->mallocFailed ); + if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null) ){ testcase( pMem->flags & MEM_Int ); testcase( pMem->flags & MEM_Real ); testcase( pMem->flags & MEM_IntReal ); - return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal); - } - if( pMem->flags & (MEM_Str|MEM_Blob) ){ - testcase( pMem->flags & MEM_Str ); - testcase( pMem->flags & MEM_Blob ); - return computeNumericType(pMem); + return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null); } + assert( pMem->flags & (MEM_Str|MEM_Blob) ); + testcase( pMem->flags & MEM_Str ); + testcase( pMem->flags & MEM_Blob ); + return computeNumericType(pMem); return 0; } From 26e817f69b4a153ca39ecb2c50fd99fd5287cf0a Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 8 Aug 2022 16:25:13 +0000 Subject: [PATCH 122/151] Avoid trying to cast an over-sized floating point value into an integer. FossilOrigin-Name: 3518cd7cb1feeefc3963da72c2d258d81d8914f1e1f427da28a00b6228cf126c --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/date.c | 2 +- src/sqliteInt.h | 1 + src/vdbe.c | 3 +-- src/vdbemem.c | 12 +++++++++++- 6 files changed, 24 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 6c2fbdb446..88135850f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\sbranch\sin\sthe\snumericType()\shelper\sroutine. -D 2022-08-08T13:04:08.230 +C Avoid\strying\sto\scast\san\sover-sized\sfloating\spoint\svalue\sinto\san\sinteger. +D 2022-08-08T16:25:13.721 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -517,7 +517,7 @@ F src/build.c 33aec2b34dbc53c3da2ea8434fccbf3c17c35ba647c3193df598cce0e861355c F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad -F src/date.c 272162554168e7af4976213850e1c4c5f33b964d299ceb0983f3d5cceba01d05 +F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 13eca2beee5b758ed033a11230971310cc4a58fcd8f6bc33cad4f677c985e96c @@ -575,7 +575,7 @@ F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d3 F src/sqlite.h.in be265991edca9aea69986758b58ba81cbf5ae403fe0c4ea1d0c9df0cdc8f25ed F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d -F src/sqliteInt.h c97b6351dba36b7bacd231610cd7173e7a7ef8469c2d9b849037415e0b1d8c79 +F src/sqliteInt.h 3ae1d20f579149c18ddd995bbeffabf036cad9f4359dc2f27dc1b778d108ff35 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4a3da6d77eeb3531cb0dbdf7047772a2a1b99f98c69e90ce009c75fe6328b2c0 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -642,13 +642,13 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 602fe229f32a96ceccae4f40824129669582096f7c355f53dbac156c9fecef23 F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 -F src/vdbe.c 6bca2e524d29d9f40e9842c287e60dbae5c428b6ea23f8ceb3114b78b3995d73 +F src/vdbe.c 81e2febe73f3db6d1ce5be29d30c4667d5fab9f6eae892ad7c80808f1af0c2bb F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c 4cfbf7ec3ed60366a38655f3f10316c5a3d68f6d4d06e462f88679392611c756 F src/vdbeaux.c 8584f4a20997fd918f0d957ab4f73d7411159772297c5020c188973bdc41dbf0 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd -F src/vdbemem.c 5ebf05c0182addedb1607ade848e1c83cef40981df94d1abfab0c59288c6064f +F src/vdbemem.c c3ce80af15e2ff5c2824a8db881681cbf511376f13613da020bac6d320c535b1 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c @@ -1981,8 +1981,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6c4fc8385ee22516e0b87fb647327ee7d1a564040ebe2a4d66fc999ef2908df2 -R e4cfac9ab2cb69853bf392410d0d10ea +P a99cc008e46ab1a4fdbe2fa87202d026a10c57be55e3e9fedd935795ed6dc694 +R bb31f7c383d6661944df8ccb04f02a45 U drh -Z 0989c0895e99c37065974c0506edc50e +Z debe4f549c31b424da3a7da3757ec640 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cbd770bafc..77c8bd5d50 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a99cc008e46ab1a4fdbe2fa87202d026a10c57be55e3e9fedd935795ed6dc694 \ No newline at end of file +3518cd7cb1feeefc3963da72c2d258d81d8914f1e1f427da28a00b6228cf126c \ No newline at end of file diff --git a/src/date.c b/src/date.c index 68d3e994f7..a3e58bc4e3 100644 --- a/src/date.c +++ b/src/date.c @@ -785,7 +785,7 @@ static int parseModifier( */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 - && (n=(int)r)==r && n>=0 && r<7 ){ + && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); p->validTZ = 0; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c6bdd3dd35..32e0e78e83 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4888,6 +4888,7 @@ int sqlite3FixSelect(DbFixer*, Select*); int sqlite3FixExpr(DbFixer*, Expr*); int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); int sqlite3RealSameAsInt(double,sqlite3_int64); +i64 sqlite3RealToI64(double); void sqlite3Int64ToText(i64,char*); int sqlite3AtoF(const char *z, double*, int, u8); int sqlite3GetInt32(const char *, int*); diff --git a/src/vdbe.c b/src/vdbe.c index dcfab51ace..131a52f14d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -315,8 +315,7 @@ static VdbeCursor *allocateCursor( */ static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ i64 iValue; - if( rValue>(double)LARGEST_INT64 || rValue<(double)SMALLEST_INT64 ) return 0; - iValue = (double)rValue; + iValue = sqlite3RealToI64(rValue); if( sqlite3RealSameAsInt(rValue,iValue) ){ *piValue = iValue; return 1; diff --git a/src/vdbemem.c b/src/vdbemem.c index fd4ab0ccc6..c6c938cb5c 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -749,6 +749,16 @@ int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ && i >= -2251799813685248LL && i < 2251799813685248LL); } +/* Convert a floating point value to its closest integer. Do so in +** a way that avoids 'outside the range of representable values' warnings +** from UBSAN. +*/ +i64 sqlite3RealToI64(double r){ + if( r<=(double)SMALLEST_INT64 ) return SMALLEST_INT64; + if( r>=(double)LARGEST_INT64) return LARGEST_INT64; + return (i64)r; +} + /* ** Convert pMem so that it has type MEM_Real or MEM_Int. ** Invalidate any prior representations. @@ -770,7 +780,7 @@ int sqlite3VdbeMemNumerify(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) - || sqlite3RealSameAsInt(pMem->u.r, (ix = (i64)pMem->u.r)) + || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r))) ){ pMem->u.i = ix; MemSetTypeFlag(pMem, MEM_Int); From 6aafd74853f7f0bf27ccbce92a69c490ddfb81d1 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Aug 2022 19:29:53 +0000 Subject: [PATCH 123/151] Avoid some cases of signed integer overflow in fts5 by casting to unsigned values. FossilOrigin-Name: 46a78c8c0ed518c4521e6e0bdebeb065bab07076abc444775002e7f4361d2242 --- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_index.c | 22 +++++++------ ext/fts5/test/fts5ubsan.test | 60 ++++++++++++++++++++++++++++++++++++ manifest | 17 +++++----- manifest.uuid | 2 +- 5 files changed, 83 insertions(+), 20 deletions(-) create mode 100644 ext/fts5/test/fts5ubsan.test diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 754f28c67f..e7e7043c60 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -286,7 +286,7 @@ void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) -#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) +#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c) #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index d2ec9ad5a9..8a29304922 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4087,7 +4087,9 @@ static void fts5WriteAppendRowid( fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); }else{ assert_nc( p->rc || iRowid>pWriter->iPrevRowid ); - fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); + fts5BufferAppendVarint(&p->rc, &pPage->buf, + (u64)iRowid - (u64)pWriter->iPrevRowid + ); } pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; @@ -4851,7 +4853,7 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ static void fts5AppendRowid( Fts5Index *p, - i64 iDelta, + u64 iDelta, Fts5Iter *pUnused, Fts5Buffer *pBuf ){ @@ -4861,7 +4863,7 @@ static void fts5AppendRowid( static void fts5AppendPoslist( Fts5Index *p, - i64 iDelta, + u64 iDelta, Fts5Iter *pMulti, Fts5Buffer *pBuf ){ @@ -4936,10 +4938,10 @@ static void fts5MergeAppendDocid( } #endif -#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ - assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ - fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ - (iLastRowid) = (iRowid); \ +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (u64)(iRowid) - (u64)(iLastRowid)); \ + (iLastRowid) = (iRowid); \ } /* @@ -5210,7 +5212,7 @@ static void fts5SetupPrefixIter( int nMerge = 1; void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); - void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); + void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ xMerge = fts5MergeRowidLists; xAppend = fts5AppendRowid; @@ -5249,7 +5251,7 @@ static void fts5SetupPrefixIter( Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; p1->xSetOutputs(p1, pSeg); if( p1->base.nData ){ - xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); iLastRowid = p1->base.iRowid; } } @@ -5297,7 +5299,7 @@ static void fts5SetupPrefixIter( iLastRowid = 0; } - xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); iLastRowid = p1->base.iRowid; } diff --git a/ext/fts5/test/fts5ubsan.test b/ext/fts5/test/fts5ubsan.test new file mode 100644 index 0000000000..2dc0aa7bd4 --- /dev/null +++ b/ext/fts5/test/fts5ubsan.test @@ -0,0 +1,60 @@ +# 2022 August 9 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test is focused on edge cases that cause ubsan errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5ubsan + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} + +set BIG 9000000000000000000 +set SMALL -9000000000000000000 + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO x1 (rowid, x) VALUES($BIG, 'aaa aba acc'); + INSERT INTO x1 (rowid, x) VALUES($SMALL, 'aaa abc acb'); + COMMIT; +} + +do_execsql_test 1.2 { + SELECT rowid, x FROM x1('ab*'); +} [list $SMALL {aaa abc acb} $BIG {aaa aba acc}] + +do_execsql_test 1.3 { + SELECT rowid, x FROM x1('ac*'); +} [list $SMALL {aaa abc acb} $BIG {aaa aba acc}] + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} + +do_execsql_test 2.1 { + INSERT INTO x1 (rowid, x) VALUES($BIG, 'aaa aba acc'); + INSERT INTO x1 (rowid, x) VALUES($SMALL, 'aaa abc acb'); +} + +do_execsql_test 2.2 { + INSERT INTO x1 (x1) VALUES('optimize'); +} + +finish_test diff --git a/manifest b/manifest index 88135850f0..f387e1e26d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\strying\sto\scast\san\sover-sized\sfloating\spoint\svalue\sinto\san\sinteger. -D 2022-08-08T16:25:13.721 +C Avoid\ssome\scases\sof\ssigned\sinteger\soverflow\sin\sfts5\sby\scasting\sto\sunsigned\svalues. +D 2022-08-08T19:29:53.445 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -130,13 +130,13 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a -F ext/fts5/fts5Int.h 36fd4a05e6e6307e3bac359a589d5f090b903afe0e7ae15db84f0ff90c79676a +F ext/fts5/fts5Int.h c0d46e399e345e35985b72a1c1af025973bfaa5b1e3563b0ce3bb0ce144a7ca3 F ext/fts5/fts5_aux.c f558e1fb9f0f86a4f7489e258c162e1f947de5ff2709087fbb465fddb7092f98 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 501e7d3566bc92766b0e11c0109a7c5a6146bc41144195459af5422f6c2078aa F ext/fts5/fts5_expr.c 40174a64829d30cc86e8266306ad24980f6911edd5ca0b8c1ce7821ea1341b88 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 3e47d9c56e4e9a6dee78bc32e006d6a28a3b5ec9ff84f3b8c381c78323201720 +F ext/fts5/fts5_index.c 4b1ac44c665667be970df780bd8e734748047bd30a971d0bb7e884af8ac6e62c F ext/fts5/fts5_main.c 6078ae86d3b813753a4f1201054550aff21a3f660e97b30f200d2b1472874151 F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -239,6 +239,7 @@ F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d2153771a8e8a20c721305a43 F ext/fts5/test/fts5trigram.test 5b4feb53a4d5aca70c841f6919c8719b5a9c805474727dda99285fccdd2e9cce +F ext/fts5/test/fts5ubsan.test 783d5a8d13ebfa169e634940228db54540780e3ba7a87ad1e4510e61440bf64b F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602 F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9 F ext/fts5/test/fts5unicode2.test 9b3df486de05fb4bde4aa7ee8de2e6dae1df6eb90e3f2e242c9383b95d314e3e @@ -1981,8 +1982,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a99cc008e46ab1a4fdbe2fa87202d026a10c57be55e3e9fedd935795ed6dc694 -R bb31f7c383d6661944df8ccb04f02a45 -U drh -Z debe4f549c31b424da3a7da3757ec640 +P 3518cd7cb1feeefc3963da72c2d258d81d8914f1e1f427da28a00b6228cf126c +R ee3441acc635032c89c4a495f8aff56d +U dan +Z 869f3966f913fbed35d61b23d360896d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 77c8bd5d50..a6b2cec1a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3518cd7cb1feeefc3963da72c2d258d81d8914f1e1f427da28a00b6228cf126c \ No newline at end of file +46a78c8c0ed518c4521e6e0bdebeb065bab07076abc444775002e7f4361d2242 \ No newline at end of file From c8e9f6818b0d1420eb6989dfd3c69192f67e1aee Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 9 Aug 2022 14:28:54 +0000 Subject: [PATCH 124/151] Omit the (undocumented) SQLITE_PCACHE_SEPARATE_HEADER compile-time option. FossilOrigin-Name: 60947da687d525e72e62bcddcdf4c5c819818855074cc318c9ee656e83897aef --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/ctime.c | 3 --- src/pcache1.c | 44 ++++++++++++++++---------------------------- tool/mkctimec.tcl | 1 - 5 files changed, 26 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index f387e1e26d..2b5d0ebe87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssome\scases\sof\ssigned\sinteger\soverflow\sin\sfts5\sby\scasting\sto\sunsigned\svalues. -D 2022-08-08T19:29:53.445 +C Omit\sthe\s(undocumented)\sSQLITE_PCACHE_SEPARATE_HEADER\scompile-time\soption. +D 2022-08-09T14:28:54.231 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -517,7 +517,7 @@ F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482 F src/build.c 33aec2b34dbc53c3da2ea8434fccbf3c17c35ba647c3193df598cce0e861355c F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c 026dbdcdbd8c3cde98a88483ee88310ff43150ab164ad768f12cc700a11495ad +F src/ctime.c 93e4b5f4faf6d3f688988a116773259a4fbfb4ddac0e9bf9d0ae0429390c2543 F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d @@ -563,7 +563,7 @@ F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564 F src/pcache.c 084e638432c610f95aea72b8509f0845d2791293f39d1b82f0c0a7e089c3bb6b F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 -F src/pcache1.c 54881292a9a5db202b2c0ac541c5e3ef9a5e8c4f1c1383adb2601d5499a60e65 +F src/pcache1.c 0b4245cd4964e635f2630908c2533cd8e9da7af3ca592e23ae8730aa25ae5eb9 F src/pragma.c 6637d624c37a8909d3edfa9d7cf694d79b49d2a0827d8c52ef15dceb641783fa F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 @@ -1909,7 +1909,7 @@ F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x -F tool/mkctimec.tcl ac96a74f5e6d9dac672d5229f79c583d3357a50e7d098e473e6b2ce2f8ae1704 x +F tool/mkctimec.tcl 8f472681f6041cf0ec26a1799feda39e55370614a6090d66fa5044f9f52d22c6 x F tool/mkkeywordhash.c 35bfc41adacc4aa6ef6fca7fd0c63e0ec0534b78daf4d0cfdebe398216bbffc3 F tool/mkmsvcmin.tcl 6ecab9fe22c2c8de4d82d4c46797bda3d2deac8e763885f5a38d0c44a895ab33 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef @@ -1982,8 +1982,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3518cd7cb1feeefc3963da72c2d258d81d8914f1e1f427da28a00b6228cf126c -R ee3441acc635032c89c4a495f8aff56d -U dan -Z 869f3966f913fbed35d61b23d360896d +P 46a78c8c0ed518c4521e6e0bdebeb065bab07076abc444775002e7f4361d2242 +R d8bef198132a525587d2454d35cbc1ca +U drh +Z a9046e25d51003cb689d3e1458ccf6d6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a6b2cec1a7..116c7715b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46a78c8c0ed518c4521e6e0bdebeb065bab07076abc444775002e7f4361d2242 \ No newline at end of file +60947da687d525e72e62bcddcdf4c5c819818855074cc318c9ee656e83897aef \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index a9391213d7..7eb43f20a7 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -683,9 +683,6 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_OMIT_XFER_OPT "OMIT_XFER_OPT", #endif -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - "PCACHE_SEPARATE_HEADER", -#endif #ifdef SQLITE_PERFORMANCE_TRACE "PERFORMANCE_TRACE", #endif diff --git a/src/pcache1.c b/src/pcache1.c index a93b146894..4f84a20b50 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -39,12 +39,13 @@ ** size can vary according to architecture, compile-time options, and ** SQLite library version number. ** -** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained -** using a separate memory allocation from the database page content. This -** seeks to overcome the "clownshoe" problem (also called "internal -** fragmentation" in academic literature) of allocating a few bytes more -** than a power of two with the memory allocator rounding up to the next -** power of two, and leaving the rounded-up space unused. +** Historical note: It used to be that if the SQLITE_PCACHE_SEPARATE_HEADER +** was defined, then the page content would be held in a separate memory +** allocation from the PgHdr1. This was intended to avoid clownshoe memory +** allocations. However, the btree layer needs a small (16-byte) overrun +** area after the page content buffer. The header serves as that overrun +** area. Therefore SQLITE_PCACHE_SEPARATE_HEADER was discontinued to avoid +** any possibility of a memory error. ** ** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates ** with this module. Information is passed back and forth as PgHdr1 pointers. @@ -89,11 +90,16 @@ typedef struct PGroup PGroup; /* ** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. +** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated +** directly before this structure and is used to cache the page content. ** -** Note: Variables isBulkLocal and isAnchor were once type "u8". That works, +** When reading a corrupt database file, it is possible that SQLite might +** read a few bytes (no more than 16 bytes) past the end of the page buffer. +** It will only read past the end of the page buffer, never write. This +** object is positioned immediately after the page buffer to serve as an +** overrun area, so that overreads are harmless. +** +** Variables isBulkLocal and isAnchor were once type "u8". That works, ** but causes a 2-byte gap in the structure for most architectures (since ** pointers must be either 4 or 8-byte aligned). As this structure is located ** in memory directly after the associated page data, if the database is @@ -438,25 +444,13 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ pcache1LeaveMutex(pCache->pGroup); #endif if( benignMalloc ){ sqlite3BeginBenignMalloc(); } -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - pPg = pcache1Alloc(pCache->szPage); - p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); - if( !pPg || !p ){ - pcache1Free(pPg); - sqlite3_free(p); - pPg = 0; - } -#else pPg = pcache1Alloc(pCache->szAlloc); -#endif if( benignMalloc ){ sqlite3EndBenignMalloc(); } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT pcache1EnterMutex(pCache->pGroup); #endif if( pPg==0 ) return 0; -#ifndef SQLITE_PCACHE_SEPARATE_HEADER p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; -#endif p->page.pBuf = pPg; p->page.pExtra = &p[1]; p->isBulkLocal = 0; @@ -480,9 +474,6 @@ static void pcache1FreePage(PgHdr1 *p){ pCache->pFree = p; }else{ pcache1Free(p->page.pBuf); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - sqlite3_free(p); -#endif } (*pCache->pnPurgeable)--; } @@ -1246,9 +1237,6 @@ int sqlite3PcacheReleaseMemory(int nReq){ && p->isAnchor==0 ){ nFree += pcache1MemSize(p->page.pBuf); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - nFree += sqlite3MemSize(p); -#endif assert( PAGE_IS_UNPINNED(p) ); pcache1PinPage(p); pcache1RemoveFromHash(p, 1); diff --git a/tool/mkctimec.tcl b/tool/mkctimec.tcl index bc723f4561..1120bc1316 100755 --- a/tool/mkctimec.tcl +++ b/tool/mkctimec.tcl @@ -254,7 +254,6 @@ set boolean_defnil_options { SQLITE_OMIT_WAL SQLITE_OMIT_WSD SQLITE_OMIT_XFER_OPT - SQLITE_PCACHE_SEPARATE_HEADER SQLITE_PERFORMANCE_TRACE SQLITE_PREFER_PROXY_LOCKING SQLITE_PROXY_DEBUG From 626bcc88dd530ce4657b5cec9c1cf6b0c56a8915 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 9 Aug 2022 16:13:21 +0000 Subject: [PATCH 125/151] Allow the name of an index to collide with a table in a different schema. FossilOrigin-Name: f963c2523872b59b8a7a14971f703f2eb0d021501b288597a958f6596885d0de --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/build.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2b5d0ebe87..1fec492199 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\s(undocumented)\sSQLITE_PCACHE_SEPARATE_HEADER\scompile-time\soption. -D 2022-08-09T14:28:54.231 +C Allow\sthe\sname\sof\san\sindex\sto\scollide\swith\sa\stable\sin\sa\sdifferent\sschema. +D 2022-08-09T16:13:21.868 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -514,7 +514,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 4a8d349b9ed4dc6d252c535227699d75319b633058a56432ebf43c9f56f9085e F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e -F src/build.c 33aec2b34dbc53c3da2ea8434fccbf3c17c35ba647c3193df598cce0e861355c +F src/build.c 1e96f35d5912a1606c9c9463dfd8eaffc76b2bc01207ee372cb3249eef5173c4 F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 93e4b5f4faf6d3f688988a116773259a4fbfb4ddac0e9bf9d0ae0429390c2543 @@ -1982,8 +1982,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 46a78c8c0ed518c4521e6e0bdebeb065bab07076abc444775002e7f4361d2242 -R d8bef198132a525587d2454d35cbc1ca +P 60947da687d525e72e62bcddcdf4c5c819818855074cc318c9ee656e83897aef +R f4e408bf2436c98ebd7456a4b6d2719e U drh -Z a9046e25d51003cb689d3e1458ccf6d6 +Z 37c870234d7cfb98492f5c9537ed0794 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 116c7715b4..6ee9ccf9af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60947da687d525e72e62bcddcdf4c5c819818855074cc318c9ee656e83897aef \ No newline at end of file +f963c2523872b59b8a7a14971f703f2eb0d021501b288597a958f6596885d0de \ No newline at end of file diff --git a/src/build.c b/src/build.c index db69b611fa..e3e79df05d 100644 --- a/src/build.c +++ b/src/build.c @@ -4036,7 +4036,7 @@ void sqlite3CreateIndex( } if( !IN_RENAME_OBJECT ){ if( !db->init.busy ){ - if( sqlite3FindTable(db, zName, 0)!=0 ){ + if( sqlite3FindTable(db, zName, pDb->zDbSName)!=0 ){ sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); goto exit_create_index; } From 8c3b7501af8fa321a1cdf3194d91603d0cbc6394 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 10 Aug 2022 09:36:10 +0000 Subject: [PATCH 126/151] wasm/fiddle refactoring part 1 of N: move fiddle app from ext/fiddle to ext/wasm/fiddle, which only contains files intended to be pushed to the live site. Disabled build of the non-fiddle wasm parts, pending a later step of the refactoring. FossilOrigin-Name: fb4eb93080288b60815be14afd7ddbbca470ce363fa3735352ea9a558fef583e --- Makefile.in | 45 +++++-------------- .../EXPORTED_FUNCTIONS.fiddle | 0 .../EXPORTED_RUNTIME_METHODS.fiddle} | 0 ext/{fiddle/Makefile => wasm/GNUmakefile} | 7 +-- ext/{fiddle/index.md => wasm/README.md} | 35 +++++++++------ ext/{ => wasm}/fiddle/emscripten.css | 0 ext/{ => wasm}/fiddle/fiddle-worker.js | 0 ext/{ => wasm}/fiddle/fiddle.html | 0 ext/{ => wasm}/fiddle/fiddle.js | 0 manifest | 28 ++++++------ manifest.uuid | 2 +- 11 files changed, 50 insertions(+), 67 deletions(-) rename ext/{fiddle => wasm}/EXPORTED_FUNCTIONS.fiddle (100%) rename ext/{fiddle/EXPORTED_RUNTIME_METHODS => wasm/EXPORTED_RUNTIME_METHODS.fiddle} (100%) rename ext/{fiddle/Makefile => wasm/GNUmakefile} (77%) rename ext/{fiddle/index.md => wasm/README.md} (78%) rename ext/{ => wasm}/fiddle/emscripten.css (100%) rename ext/{ => wasm}/fiddle/fiddle-worker.js (100%) rename ext/{ => wasm}/fiddle/fiddle.html (100%) rename ext/{ => wasm}/fiddle/fiddle.js (100%) diff --git a/Makefile.in b/Makefile.in index cfa81cacf9..3b9a7b56bc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1524,13 +1524,12 @@ sqlite3.dll: $(REAL_LIBOBJ) sqlite3.def # # fiddle/wasm section # -fiddle_dir = ext/fiddle -fiddle_dir_abs = $(TOP)/$(fiddle_dir) +wasm_dir = ext/wasm +wasm_dir_abs = $(TOP)/ext/wasm # ^^^ some emcc opts require absolute paths -fiddle_html = $(fiddle_dir)/fiddle.html +fiddle_dir = $(wasm_dir)/fiddle +fiddle_dir_abs = $(TOP)/$(fiddle_dir) fiddle_module_js = $(fiddle_dir)/fiddle-module.js -sqlite3_wasm_js = $(fiddle_dir)/sqlite3.js -sqlite3_wasm = $(fiddle_dir)/sqlite3.wasm #emcc_opt = -O0 #emcc_opt = -O1 #emcc_opt = -O2 @@ -1542,54 +1541,34 @@ emcc_flags = $(emcc_opt) \ -sSTRICT_JS \ -sENVIRONMENT=web \ -sMODULARIZE \ - -sEXPORTED_RUNTIME_METHODS=@$(fiddle_dir_abs)/EXPORTED_RUNTIME_METHODS \ + -sEXPORTED_RUNTIME_METHODS=@$(wasm_dir_abs)/EXPORTED_RUNTIME_METHODS.fiddle \ -sDYNAMIC_EXECUTION=0 \ --minify 0 \ - -I. $(SHELL_OPT) + -I. $(SHELL_OPT) \ + -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_UTF16 -DSQLITE_OMIT_DEPRECATED $(fiddle_module_js): Makefile sqlite3.c shell.c \ - $(fiddle_dir)/EXPORTED_RUNTIME_METHODS \ - $(fiddle_dir)/EXPORTED_FUNCTIONS.fiddle + $(wasm_dir)/EXPORTED_RUNTIME_METHODS.fiddle \ + $(wasm_dir)/EXPORTED_FUNCTIONS.fiddle emcc -o $@ $(emcc_flags) \ -sEXPORT_NAME=initFiddleModule \ - -sEXPORTED_FUNCTIONS=@$(fiddle_dir_abs)/EXPORTED_FUNCTIONS.fiddle \ + -sEXPORTED_FUNCTIONS=@$(wasm_dir_abs)/EXPORTED_FUNCTIONS.fiddle \ -DSQLITE_SHELL_FIDDLE \ sqlite3.c shell.c gzip < $@ > $@.gz gzip < $(fiddle_dir)/fiddle-module.wasm > $(fiddle_dir)/fiddle-module.wasm.gz -$(sqlite3_wasm_js): Makefile sqlite3.c $(fiddle_dir)/wasm_util.c \ - $(fiddle_dir)/sqlite3-api.js \ - $(fiddle_dir)/EXPORTED_RUNTIME_METHODS \ - $(fiddle_dir)/EXPORTED_FUNCTIONS.sqlite3-api - emcc -o $@ $(emcc_flags) \ - -sEXPORT_NAME=sqlite3InitModule \ - -sEXPORTED_FUNCTIONS=@$(fiddle_dir_abs)/EXPORTED_FUNCTIONS.sqlite3-api \ - --post-js=$(fiddle_dir)/sqlite3-api.js \ - --no-entry \ - sqlite3.c $(fiddle_dir)/wasm_util.c - gzip < $@ > $@.gz - gzip < $(sqlite3_wasm) > $(sqlite3_wasm).gz - gzip < $(fiddle_dir)/sqlite3-api.js > $(fiddle_dir)/sqlite3-api.js.gz $(fiddle_dir)/fiddle.js.gz: $(fiddle_dir)/fiddle.js gzip < $< > $@ -$(fiddle_dir)/sqlite3-api.js.gz: $(fiddle_dir)/sqlite3-api.js - gzip < $< > $@ fiddle_generated = $(fiddle_module_js) $(fiddle_module_js).gz \ $(fiddle_dir)/fiddle-module.wasm \ $(fiddle_dir)/fiddle-module.wasm.gz \ $(fiddle_dir)/fiddle.js.gz -sqlite3_wasm_generated = \ - $(sqlite3_wasm) $(sqlite3_wasm).gz \ - $(sqlite3_wasm_js) $(sqlite3_wasm_js).gz \ - $(fiddle_dir)/sqlite3.js.gz \ - $(fiddle_dir)/sqlite3-api.js.gz clean-wasm: - rm -f $(fiddle_generated) $(sqlite3_wasm_generated) + rm -f $(fiddle_generated) clean: clean-wasm fiddle: $(fiddle_module_js) $(fiddle_dir)/fiddle.js.gz -sqlite3-wasm: $(sqlite3_wasm_js) -wasm: fiddle sqlite3-wasm +wasm: fiddle ######################################################################## # Explanation of the emcc build flags follows. Full docs for these can # be found at: diff --git a/ext/fiddle/EXPORTED_FUNCTIONS.fiddle b/ext/wasm/EXPORTED_FUNCTIONS.fiddle similarity index 100% rename from ext/fiddle/EXPORTED_FUNCTIONS.fiddle rename to ext/wasm/EXPORTED_FUNCTIONS.fiddle diff --git a/ext/fiddle/EXPORTED_RUNTIME_METHODS b/ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle similarity index 100% rename from ext/fiddle/EXPORTED_RUNTIME_METHODS rename to ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle diff --git a/ext/fiddle/Makefile b/ext/wasm/GNUmakefile similarity index 77% rename from ext/fiddle/Makefile rename to ext/wasm/GNUmakefile index b274574d7f..99a1c5b031 100644 --- a/ext/fiddle/Makefile +++ b/ext/wasm/GNUmakefile @@ -6,11 +6,6 @@ default: clean: $(MAKE) -C ../../ clean-wasm -fiddle_files = emscripten.css fiddle.html \ - fiddle.js fiddle-module.js \ - fiddle-module.wasm fiddle-worker.js \ - $(wildcard *.wasm.gz) $(wildcard *.js.gz) - # fiddle_remote is the remote destination for the fiddle app. It # must be a [user@]HOST:/path for rsync. # Note that the target "should probably" contain a symlink of @@ -31,4 +26,4 @@ push-fiddle: $(fiddle_files) echo "fiddle_remote must be a [user@]HOST:/path for rsync"; \ exit 1; \ fi - rsync -va $(fiddle_files) $(fiddle_remote) + rsync -va fiddle/ $(fiddle_remote) diff --git a/ext/fiddle/index.md b/ext/wasm/README.md similarity index 78% rename from ext/fiddle/index.md rename to ext/wasm/README.md index 9d1f8d83ea..1702e1f427 100644 --- a/ext/fiddle/index.md +++ b/ext/wasm/README.md @@ -1,7 +1,5 @@ -This directory houses a "fiddle"-style application which embeds a -[Web Assembly (WASM)](https://en.wikipedia.org/wiki/WebAssembly) -build of the sqlite3 shell app into an HTML page, effectively running -the shell in a client-side browser. +This directory houses the [Web Assembly (WASM)](https://en.wikipedia.org/wiki/WebAssembly) +parts of the sqlite3 build. It requires [emscripten][] and that the build environment be set up for emscripten. A mini-HOWTO for setting that up follows... @@ -22,8 +20,15 @@ $ ./emsdk install latest $ ./emsdk activate latest ``` -Those parts only need to be run once. The following needs to be run for each -shell instance which needs the `emcc` compiler: +Those parts only need to be run once, but the SDK can be updated using: + +``` +$ git pull +$ ./emsdk activate latest +``` + +The following needs to be run for each shell instance which needs the +`emcc` compiler: ``` # Activate PATH and other environment variables in the current terminal: @@ -33,8 +38,11 @@ $ which emcc /path/to/emsdk/upstream/emscripten/emcc ``` -That `env` script needs to be sourced for building this application from the -top of the sqlite3 build tree: +Optionally, add that to your login shell's resource file (`~/.bashrc` +or equivalent). + +That `env` script needs to be sourced for building this application +from the top of the sqlite3 build tree: ``` $ make fiddle @@ -43,29 +51,30 @@ $ make fiddle Or: ``` -$ cd ext/fiddle +$ cd ext/wasm $ make ``` That will generate the fiddle application under -[ext/fiddle](/dir/ext/fiddle), as `fiddle.html`. That application +[ext/fiddle](/dir/ext/wasm/fiddle), as `fiddle.html`. That application cannot, due to XMLHttpRequest security limitations, run if the HTML file is opened directly in the browser (i.e. if it is opened using a `file://` URL), so it needs to be served via an HTTP server. For example, using [althttpd][]: ``` -$ cd ext/fiddle -$ althttpd -debug 1 -jail 0 -port 9090 -root . +$ cd ext/wasm/fiddle +$ althttpd -page fiddle.html ``` -Then browse to `http://localhost:9090/fiddle.html`. +That will open the system's browser and run the fiddle app's page. Note that when serving this app via [althttpd][], it must be a version from 2022-05-17 or newer so that it recognizes the `.wasm` file extension and responds with the mimetype `application/wasm`, as the WASM loader is pedantic about that detail. + # Known Quirks and Limitations Some "impedence mismatch" between C and WASM/JavaScript is to be diff --git a/ext/fiddle/emscripten.css b/ext/wasm/fiddle/emscripten.css similarity index 100% rename from ext/fiddle/emscripten.css rename to ext/wasm/fiddle/emscripten.css diff --git a/ext/fiddle/fiddle-worker.js b/ext/wasm/fiddle/fiddle-worker.js similarity index 100% rename from ext/fiddle/fiddle-worker.js rename to ext/wasm/fiddle/fiddle-worker.js diff --git a/ext/fiddle/fiddle.html b/ext/wasm/fiddle/fiddle.html similarity index 100% rename from ext/fiddle/fiddle.html rename to ext/wasm/fiddle/fiddle.html diff --git a/ext/fiddle/fiddle.js b/ext/wasm/fiddle/fiddle.js similarity index 100% rename from ext/fiddle/fiddle.js rename to ext/wasm/fiddle/fiddle.js diff --git a/manifest b/manifest index d312d2b6ed..913da59c3c 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Merged\sin\strunk\sfor\spending\stree\srefactoring. -D 2022-08-10T07:58:57.865 +C wasm/fiddle\srefactoring\spart\s1\sof\sN:\smove\sfiddle\sapp\sfrom\sext/fiddle\sto\sext/wasm/fiddle,\swhich\sonly\scontains\sfiles\sintended\sto\sbe\spushed\sto\sthe\slive\ssite.\sDisabled\sbuild\sof\sthe\snon-fiddle\swasm\sparts,\spending\sa\slater\sstep\sof\sthe\srefactoring. +D 2022-08-10T09:36:10.232 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in efc6e3d5c558ef5abb20cad20cf96bcecb86755ce45a858e6c64bfbd9749fd30 +F Makefile.in a77d419b19eb2f806109ae2d0b81abb39a3a8659b00e528da7e27bd95c7e29fd F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -55,16 +55,8 @@ F ext/expert/expert1.test 3c642a4e7bbb14f21ddab595436fb465a4733f47a0fe5b2855e1d5 F ext/expert/sqlite3expert.c 6ca30d73b9ed75bd56d6e0d7f2c962d2affaa72c505458619d0ff5d9cdfac204 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 -F ext/fiddle/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api 356c356931b58eccf68367120f304db43ab6c2ef2f62f17f12f5a99737b43c38 -F ext/fiddle/EXPORTED_RUNTIME_METHODS a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 -F ext/fiddle/Makefile 1d303ee6449be3bab67f4b1456bacf903eb1c5e6d40d3b391651d7e0879d891a F ext/fiddle/SqliteTestUtil.js 2e87d424b12674476bdf8139934dcacc3ff8a7a5f5ff4392ba5e5a8d8cee9fbd -F ext/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae -F ext/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 -F ext/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 -F ext/fiddle/index.md d9c1c308d8074341bc3b11d1d39073cd77754cb3ca9aeb949f23fdd8323d81cf F ext/fiddle/sqlite3-api.js 5a6cc120f3eeaab65e49bcdab234e83d83c67440e04bd97191bdc004ac0cda35 F ext/fiddle/sqlite3-worker.js 50b7a9ce14c8fae0af965e35605fe12cafb79c1e01e99216d8110d8b02fbf4b5 F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a @@ -490,6 +482,14 @@ F ext/session/test_session.c f433f68a8a8c64b0f5bc74dc725078f12483301ad4ae8375205 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb +F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 w ext/fiddle/EXPORTED_FUNCTIONS.fiddle +F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 w ext/fiddle/EXPORTED_RUNTIME_METHODS +F ext/wasm/GNUmakefile c71257754d3f69ed19308e91c2829be98532aa27ba1feaefe53d2bf17c047dc8 w ext/fiddle/Makefile +F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 w ext/fiddle/index.md +F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f w ext/fiddle/emscripten.css +F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae w ext/fiddle/fiddle-worker.js +F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 w ext/fiddle/fiddle.html +F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 w ext/fiddle/fiddle.js F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1983,8 +1983,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d662796c658997be13fdc3b77ad97101b9513da53fd0b824d7a4050cac3f7eba f963c2523872b59b8a7a14971f703f2eb0d021501b288597a958f6596885d0de -R 32a46e2655367bd84dca87f379c92af7 +P c3a3cb0103126210692bbeb703e7b8793974042e1fc2473be6d0a0d9b07d5770 +R 7933790875a6b3f54b530ce22fe4d8f0 U stephan -Z ae9739b616b9feb7c2ad647c284be5e7 +Z f8f037574e62bc4aeafe6c26802ca165 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b2f40f826a..1031463cb5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3a3cb0103126210692bbeb703e7b8793974042e1fc2473be6d0a0d9b07d5770 \ No newline at end of file +fb4eb93080288b60815be14afd7ddbbca470ce363fa3735352ea9a558fef583e \ No newline at end of file From 3961b2636329819f16b1f32739274f3a1d3273de Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 10 Aug 2022 11:26:08 +0000 Subject: [PATCH 127/151] wasm refactoring part 2 of (apparently) 2: moved ext/fiddle/... into ext/wasm and restructured the core API-related parts of the JS/WASM considerably. FossilOrigin-Name: 27f9da4eaaff39d1d58e9ffef7ddccf1e41b3726914f754b920e3e1fb572cba6 --- Makefile.in | 4 +- ext/fiddle/SqliteTestUtil.js | 144 -- ext/fiddle/sqlite3-api.js | 2068 ----------------- ext/fiddle/sqlite3-worker.js | 44 - ext/fiddle/testing1.js | 239 -- ext/fiddle/testing2.js | 235 -- ext/fiddle/wasm_util.c | 127 - ext/wasm/GNUmakefile | 270 ++- .../api}/EXPORTED_FUNCTIONS.sqlite3-api | 21 +- .../api/EXPORTED_RUNTIME_METHODS.sqlite3-api | 3 + ext/wasm/api/post-js-footer.js | 3 + ext/wasm/api/post-js-header.js | 26 + ext/wasm/api/sqlite3-api-cleanup.js | 44 + ext/wasm/api/sqlite3-api-glue.js | 211 ++ ext/wasm/api/sqlite3-api-oo1.js | 1438 ++++++++++++ ext/wasm/api/sqlite3-api-opfs.js | 394 ++++ ext/wasm/api/sqlite3-api-prologue.js | 593 +++++ ext/wasm/api/sqlite3-api-worker.js | 421 ++++ ext/wasm/api/sqlite3-wasi.h | 69 + ext/wasm/api/sqlite3-wasm.c | 413 ++++ ext/wasm/api/sqlite3-worker.js | 31 + ext/wasm/common/SqliteTestUtil.js | 173 ++ ext/wasm/common/emscripten.css | 24 + ext/{fiddle => wasm/common}/testing.css | 1 + ext/wasm/common/whwasmutil.js | 1548 ++++++++++++ ext/wasm/jaccwabyt/jaccwabyt.js | 737 ++++++ ext/wasm/jaccwabyt/jaccwabyt.md | 1078 +++++++++ ext/wasm/jaccwabyt/jaccwabyt_test.c | 178 ++ ext/wasm/jaccwabyt/jaccwabyt_test.exports | 10 + ext/{fiddle => wasm}/testing1.html | 13 +- ext/wasm/testing1.js | 1088 +++++++++ ext/{fiddle => wasm}/testing2.html | 11 +- ext/wasm/testing2.js | 340 +++ manifest | 63 +- manifest.uuid | 2 +- 35 files changed, 9162 insertions(+), 2902 deletions(-) delete mode 100644 ext/fiddle/SqliteTestUtil.js delete mode 100644 ext/fiddle/sqlite3-api.js delete mode 100644 ext/fiddle/sqlite3-worker.js delete mode 100644 ext/fiddle/testing1.js delete mode 100644 ext/fiddle/testing2.js delete mode 100644 ext/fiddle/wasm_util.c rename ext/{fiddle => wasm/api}/EXPORTED_FUNCTIONS.sqlite3-api (71%) create mode 100644 ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api create mode 100644 ext/wasm/api/post-js-footer.js create mode 100644 ext/wasm/api/post-js-header.js create mode 100644 ext/wasm/api/sqlite3-api-cleanup.js create mode 100644 ext/wasm/api/sqlite3-api-glue.js create mode 100644 ext/wasm/api/sqlite3-api-oo1.js create mode 100644 ext/wasm/api/sqlite3-api-opfs.js create mode 100644 ext/wasm/api/sqlite3-api-prologue.js create mode 100644 ext/wasm/api/sqlite3-api-worker.js create mode 100644 ext/wasm/api/sqlite3-wasi.h create mode 100644 ext/wasm/api/sqlite3-wasm.c create mode 100644 ext/wasm/api/sqlite3-worker.js create mode 100644 ext/wasm/common/SqliteTestUtil.js create mode 100644 ext/wasm/common/emscripten.css rename ext/{fiddle => wasm/common}/testing.css (93%) create mode 100644 ext/wasm/common/whwasmutil.js create mode 100644 ext/wasm/jaccwabyt/jaccwabyt.js create mode 100644 ext/wasm/jaccwabyt/jaccwabyt.md create mode 100644 ext/wasm/jaccwabyt/jaccwabyt_test.c create mode 100644 ext/wasm/jaccwabyt/jaccwabyt_test.exports rename ext/{fiddle => wasm}/testing1.html (76%) create mode 100644 ext/wasm/testing1.js rename ext/{fiddle => wasm}/testing2.html (79%) create mode 100644 ext/wasm/testing2.js diff --git a/Makefile.in b/Makefile.in index 3b9a7b56bc..193095c681 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1564,9 +1564,9 @@ fiddle_generated = $(fiddle_module_js) $(fiddle_module_js).gz \ $(fiddle_dir)/fiddle-module.wasm.gz \ $(fiddle_dir)/fiddle.js.gz -clean-wasm: +clean-fiddle: rm -f $(fiddle_generated) -clean: clean-wasm +clean: clean-fiddle fiddle: $(fiddle_module_js) $(fiddle_dir)/fiddle.js.gz wasm: fiddle ######################################################################## diff --git a/ext/fiddle/SqliteTestUtil.js b/ext/fiddle/SqliteTestUtil.js deleted file mode 100644 index bcbdc59c51..0000000000 --- a/ext/fiddle/SqliteTestUtil.js +++ /dev/null @@ -1,144 +0,0 @@ -/* - 2022-05-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file contains bootstrapping code used by various test scripts - which live in this file's directory. -*/ -(function(){ - /* querySelectorAll() proxy */ - const EAll = function(/*[element=document,] cssSelector*/){ - return (arguments.length>1 ? arguments[0] : document) - .querySelectorAll(arguments[arguments.length-1]); - }; - /* querySelector() proxy */ - const E = function(/*[element=document,] cssSelector*/){ - return (arguments.length>1 ? arguments[0] : document) - .querySelector(arguments[arguments.length-1]); - }; - - /** - Helpers for writing sqlite3-specific tests. - */ - self/*window or worker*/.SqliteTestUtil = { - /** Running total of the number of tests run via - this API. */ - counter: 0, - /** - If expr is a function, it is called and its result - is returned, coerced to a bool, else expr, coerced to - a bool, is returned. - */ - toBool: function(expr){ - return (expr instanceof Function) ? !!expr() : !!expr; - }, - /** abort() if expr is false. If expr is a function, it - is called and its result is evaluated. - */ - assert: function f(expr, msg){ - if(!f._){ - f._ = ('undefined'===typeof abort - ? (msg)=>{throw new Error(msg)} - : abort); - } - ++this.counter; - if(!this.toBool(expr)){ - f._(msg || "Assertion failed."); - } - return this; - }, - /** Identical to assert() but throws instead of calling - abort(). */ - affirm: function(expr, msg){ - ++this.counter; - if(!this.toBool(expr)) throw new Error(msg || "Affirmation failed."); - return this; - }, - /** Calls f() and squelches any exception it throws. If it - does not throw, this function throws. */ - mustThrow: function(f, msg){ - ++this.counter; - let err; - try{ f(); } catch(e){err=e;} - if(!err) throw new Error(msg || "Expected exception."); - return this; - }, - /** Throws if expr is truthy or expr is a function and expr() - returns truthy. */ - throwIf: function(expr, msg){ - ++this.counter; - if(this.toBool(expr)) throw new Error(msg || "throwIf() failed"); - return this; - }, - /** Throws if expr is falsy or expr is a function and expr() - returns falsy. */ - throwUnless: function(expr, msg){ - ++this.counter; - if(!this.toBool(expr)) throw new Error(msg || "throwUnless() failed"); - return this; - } - }; - - - /** - This is a module object for use with the emscripten-installed - sqlite3InitModule() factory function. - */ - self.sqlite3TestModule = { - postRun: [ - /* function(theModule){...} */ - ], - //onRuntimeInitialized: function(){}, - /* Proxy for C-side stdout output. */ - print: function(){ - console.log.apply(console, Array.prototype.slice.call(arguments)); - }, - /* Proxy for C-side stderr output. */ - printErr: function(){ - console.error.apply(console, Array.prototype.slice.call(arguments)); - }, - /** - Called by the module init bits to report loading - progress. It gets passed an empty argument when loading is - done (after onRuntimeInitialized() and any this.postRun - callbacks have been run). - */ - setStatus: function f(text){ - if(!f.last){ - f.last = { text: '', step: 0 }; - f.ui = { - status: E('#module-status'), - progress: E('#module-progress'), - spinner: E('#module-spinner') - }; - } - if(text === f.last.text) return; - f.last.text = text; - if(f.ui.progress){ - f.ui.progress.value = f.last.step; - f.ui.progress.max = f.last.step + 1; - } - ++f.last.step; - if(text) { - f.ui.status.classList.remove('hidden'); - f.ui.status.innerText = text; - }else{ - if(f.ui.progress){ - f.ui.progress.remove(); - f.ui.spinner.remove(); - delete f.ui.progress; - delete f.ui.spinner; - } - f.ui.status.classList.add('hidden'); - } - } - }; -})(self/*window or worker*/); diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js deleted file mode 100644 index 01eb4c57fd..0000000000 --- a/ext/fiddle/sqlite3-api.js +++ /dev/null @@ -1,2068 +0,0 @@ -/* - 2022-05-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file is intended to be appended to the emcc-generated - sqlite3.js via emcc: - - emcc ... -sMODULARIZE -sEXPORT_NAME=sqlite3InitModule --post-js=THIS_FILE - - It is loaded by importing the emcc-generated sqlite3.js, then: - - sqlite3InitModule({module object}).then( - function(theModule){ - theModule.sqlite3 == an object containing this file's - deliverables: - { - api: bindings for much of the core sqlite3 APIs, - SQLite3: high-level OO API wrapper - } - }); - - It is up to the caller to provide a module object compatible with - emcc, but it can be a plain empty object. The object passed to - sqlite3InitModule() will get populated by the emscripten-generated - bits and, in part, by the code from this file. Specifically, this file - installs the `theModule.sqlite3` part shown above. - - The resulting sqlite3.api object wraps the standard sqlite3 C API in - a way as close to its native form as JS allows for. The - sqlite3.SQLite3 object provides a higher-level wrapper more - appropriate for general client-side use in JS. - - Because using certain parts of the low-level API properly requires - some degree of WASM-related magic, it is not recommended that that - API be used as-is in client-level code. Rather, client code is - encouraged use the higher-level OO API or write a custom wrapper on - top of the lower-level API. In short, most of the C-style API is - used in an intuitive manner from JS but any C-style APIs which take - pointers-to-pointer arguments require WASM-specific interfaces - installed by Emscripten-generated code. Those which take or return - only integers, doubles, strings, or "plain" pointers to db or - statement objects can be used in "as normal," noting that "pointers" - in WASM are simply 32-bit integers. - - - Specific goals of this project: - - - Except where noted in the non-goals, provide a more-or-less - complete wrapper to the sqlite3 C API, insofar as WASM feature - parity with C allows for. In fact, provide at least 3... - - 1) Bind a low-level sqlite3 API which is as close to the native - one as feasible in terms of usage. - - 2) A higher-level API, more akin to sql.js and node.js-style - implementations. This one speaks directly to the low-level - API. This API must be used from the same thread as the - low-level API. - - 3) A second higher-level API which speaks to the previous APIs via - worker messages. This one is intended for use in the main - thread, with the lower-level APIs installed in a Worker thread, - and talking to them via Worker messages. Because Workers are - asynchronouns and have only a single message channel, some - acrobatics are needed here to feed async work results back to - the client (as we cannot simply pass around callbacks between - the main and Worker threads). - - - Insofar as possible, support client-side storage using JS - filesystem APIs. As of this writing, such things are still very - much TODO. - - - Specific non-goals of this project: - - - As WASM is a web-centric technology and UTF-8 is the King of - Encodings in that realm, there are no currently plans to support - the UTF16-related sqlite3 APIs. They would add a complication to - the bindings for no appreciable benefit. - - - Supporting old or niche-market platforms. WASM is built for a - modern web and requires modern platforms. - - - Attribution: - - Though this code is not a direct copy/paste, much of the - functionality in this file is strongly influenced by the - corresponding features in sql.js: - - https://github.com/sql-js/sql.js - - sql.js was an essential stepping stone in this code's development as - it demonstrated how to handle some of the WASM-related voodoo (like - handling pointers-to-pointers and adding JS implementations of - C-bound callback functions). These APIs have a considerably - different shape than sql.js's, however. -*/ -if(!Module.postRun) Module.postRun = []; -/* ^^^^ the name Module is, in this setup, scope-local in the generated - file sqlite3.js, with which this file gets combined at build-time. */ -Module.postRun.push(function(namespace/*the module object, the target for - installed features*/){ - 'use strict'; - /** - */ - const SQM/*interal-use convenience alias*/ = namespace/*the sqlite module object */; - - /** Throws a new Error, the message of which is the concatenation - all args with a space between each. */ - const toss = function(){ - throw new Error(Array.prototype.join.call(arguments, ' ')); - }; - - /** Returns true if n is a 32-bit (signed) integer, else false. */ - const isInt32 = function(n){ - return !!(n===(n|0) && n<=2147483647 && n>=-2147483648); - }; - - /** Returns v if v appears to be a TypedArray, else false. */ - const isTypedArray = (v)=>{ - return (v && v.constructor && isInt32(v.constructor.BYTES_PER_ELEMENT)) ? v : false; - }; - - /** - Returns true if v appears to be one of our bind()-able - TypedArray types: Uint8Array or Int8Array. Support for - TypedArrays with element sizes >1 is a potential TODO. - */ - const isBindableTypedArray = (v)=>{ - return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); - }; - - /** - Returns true if v appears to be one of the TypedArray types - which is legal for holding SQL code (as opposed to binary blobs). - - Currently this is the same as isBindableTypedArray() but it - seems likely that we'll eventually want to add Uint32Array - and friends to the isBindableTypedArray() list but not to the - isSQLableTypedArray() list. - */ - const isSQLableTypedArray = isBindableTypedArray; - - /** Returns true if isBindableTypedArray(v) does, else throws with a message - that v is not a supported TypedArray value. */ - const affirmBindableTypedArray = (v)=>{ - return isBindableTypedArray(v) - || toss("Value is not of a supported TypedArray type."); - }; - - /** - The main sqlite3 binding API gets installed into this object, - mimicking the C API as closely as we can. The numerous members - names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as - possible, identically to the C-native counterparts, as documented at: - - https://www.sqlite.org/c3ref/intro.html - - A very few exceptions require an additional level of proxy - function or may otherwise require special attention in the WASM - environment, and all such cases are document here. Those not - documented here are installed as 1-to-1 proxies for their C-side - counterparts. - */ - const api = { - /** - When using sqlite3_open_v2() it is important to keep the following - in mind: - - https://www.sqlite.org/c3ref/open.html - - - The flags for use with its 3rd argument are installed in this - object using the C-cide names, e.g. SQLITE_OPEN_CREATE. - - - If the combination of flags passed to it are invalid, - behavior is undefined. Thus is is never okay to call this - with fewer than 3 arguments, as JS will default the - missing arguments to `undefined`, which will result in a - flag value of 0. Most of the available SQLITE_OPEN_xxx - flags are meaningless in the WASM build, e.g. the mutext- - and cache-related flags, but they are retained in this - API for consistency's sake. - - - The final argument to this function specifies the VFS to - use, which is largely (but not entirely!) meaningless in - the WASM environment. It should always be null or - undefined, and it is safe to elide that argument when - calling this function. - */ - sqlite3_open_v2: function(filename,dbPtrPtr,flags,vfsStr){}/*installed later*/, - - /** - The sqlite3_prepare_v2() binding handles two different uses - with differing JS/WASM semantics: - - 1) sqlite3_prepare_v2(pDb, sqlString, -1, ppStmt [, null]) - - 2) sqlite3_prepare_v2(pDb, sqlPointer, -1, ppStmt, sqlPointerToPointer) - - Note that the SQL length argument (the 3rd argument) must - always be negative because it must be a byte length and - that value is expensive to calculate from JS (where only - the character length of strings is readily available). It - is retained in this API's interface for code/documentation - compatibility reasons but is currently _always_ - ignored. When using the 2nd form of this call, it is - critical that the custom-allocated string be terminated - with a 0 byte. (Potential TODO: if the 3rd argument is >0, - assume the caller knows precisely what they're doing, vis a - vis WASM memory management, and pass it on as-is. That - approach currently seems fraught with peril.) - - In usage (1), the 2nd argument must be of type string, - Uint8Array, or Int8Array (either of which is assumed to - hold SQL). If it is, this function assumes case (1) and - calls the underyling C function with: - - (pDb, sqlAsString, -1, ppStmt, null) - - The pzTail argument is ignored in this case because its result - is meaningless when a string-type value is passed through - (because the string goes through another level of internal - conversion for WASM's sake and the result pointer would refer - to that conversion's memory, not the passed-in string). - - If sql is not a string or supported TypedArray, it must be - a _pointer_ to a string which was allocated via - api.wasm.allocateUTF8OnStack(), api.wasm._malloc(), or - equivalent. In that case, the final argument may be - 0/null/undefined or must be a pointer to which the "tail" - of the compiled SQL is written, as documented for the - C-side sqlite3_prepare_v2(). In case (2), the underlying C - function is called with: - - (pDb, sqlAsPointer, -1, ppStmt, pzTail) - - It returns its result and compiled statement as documented - in the C API. Fetching the output pointers (4th and 5th - parameters) requires using api.wasm.getValue() and the - pzTail will point to an address relative to the - sqlAsPointer value. - - If passed an invalid 2nd argument type, this function will - throw. That behavior is in strong constrast to all of the - other C-bound functions (which return a non-0 result code - on error) but is necessary because we have to way to set - the db's error state such that this function could return a - non-0 integer and the client could call sqlite3_errcode() - or sqlite3_errmsg() to fetch it. - */ - sqlite3_prepare_v2: function(dbPtr, sql, sqlByteLen, stmtPtrPtr, strPtrPtr){}/*installed later*/, - - /** - Holds state which are specific to the WASM-related - infrastructure and glue code. It is not expected that client - code will normally need these, but they're exposed here in case it - does. - - Note that a number of members of this object are injected - dynamically after the api object is fully constructed, so - not all are documented inline here. - */ - wasm: { - /** - api.wasm._malloc()'s srcTypedArray.byteLength bytes, - populates them with the values from the source - TypedArray, and returns the pointer to that memory. The - returned pointer must eventually be passed to - api.wasm._free() to clean it up. - - As a special case, to avoid further special cases where - this is used, if srcTypedArray.byteLength is 0, it - allocates a single byte and sets it to the value - 0. Even in such cases, calls must behave as if the - allocated memory has exactly srcTypedArray.byteLength - bytes. - - ACHTUNG: this currently only works for Uint8Array and - Int8Array types and will throw if srcTypedArray is of - any other type. - */ - mallocFromTypedArray: function(srcTypedArray){ - affirmBindableTypedArray(srcTypedArray); - const pRet = api.wasm._malloc(srcTypedArray.byteLength || 1); - this.heapForSize(srcTypedArray).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); - return pRet; - }, - /** Convenience form of this.heapForSize(8,false). */ - HEAP8: ()=>SQM['HEAP8'], - /** Convenience form of this.heapForSize(8,true). */ - HEAPU8: ()=>SQM['HEAPU8'], - /** - Requires n to be one of (8, 16, 32) or a TypedArray - instance of Int8Array, Int16Array, Int32Array, or their - Uint counterparts. - - Returns the current integer-based TypedArray view of - the WASM heap memory buffer associated with the given - block size. If unsigned is truthy then the "U" - (unsigned) variant of that view is returned, else the - signed variant is returned. If passed a TypedArray - value and no 2nd argument then the 2nd argument - defaults to the signedness of that array. Note that - Float32Array and Float64Array views are not supported - by this function. - - Note that growth of the heap will invalidate any - references to this heap, so do not hold a reference - longer than needed and do not use a reference - after any operation which may allocate. - - Throws if passed an invalid n - */ - heapForSize: function(n,unsigned = true){ - if(isTypedArray(n)){ - if(1===arguments.length){ - unsigned = n instanceof Uint8Array || n instanceof Uint16Array - || n instanceof Uint32Array; - } - n = n.constructor.BYTES_PER_ELEMENT * 8; - } - switch(n){ - case 8: return SQM[unsigned ? 'HEAPU8' : 'HEAP8']; - case 16: return SQM[unsigned ? 'HEAPU16' : 'HEAP16']; - case 32: return SQM[unsigned ? 'HEAPU32' : 'HEAP32']; - } - toss("Invalid heapForSize() size: expecting 8, 16, or 32."); - } - } - }; - [/* C-side functions to bind. Each entry is an array with 3 elements: - - ["c-side name", - "result type" (cwrap() syntax), - [arg types in cwrap() syntax] - ] - */ - ["sqlite3_bind_blob","number",["number", "number", "number", "number", "number"]], - ["sqlite3_bind_double","number",["number", "number", "number"]], - ["sqlite3_bind_int","number",["number", "number", "number"]], - /*Noting that JS/wasm combo does not currently support 64-bit integers: - ["sqlite3_bind_int64","number",["number", "number", "number"]],*/ - ["sqlite3_bind_null","void",["number"]], - ["sqlite3_bind_parameter_count", "number", ["number"]], - ["sqlite3_bind_parameter_index","number",["number", "string"]], - ["sqlite3_bind_text","number",["number", "number", "number", "number", "number"]], - ["sqlite3_changes", "number", ["number"]], - ["sqlite3_clear_bindings","number",["number"]], - ["sqlite3_close_v2", "number", ["number"]], - ["sqlite3_column_blob","number", ["number", "number"]], - ["sqlite3_column_bytes","number",["number", "number"]], - ["sqlite3_column_count", "number", ["number"]], - ["sqlite3_column_count","number",["number"]], - ["sqlite3_column_double","number",["number", "number"]], - ["sqlite3_column_int","number",["number", "number"]], - /*Noting that JS/wasm combo does not currently support 64-bit integers: - ["sqlite3_column_int64","number",["number", "number"]],*/ - ["sqlite3_column_name","string",["number", "number"]], - ["sqlite3_column_text","string",["number", "number"]], - ["sqlite3_column_type","number",["number", "number"]], - ["sqlite3_compileoption_get", "string", ["number"]], - ["sqlite3_compileoption_used", "number", ["string"]], - ["sqlite3_create_function_v2", "number", - ["number", "string", "number", "number","number", - "number", "number", "number", "number"]], - ["sqlite3_data_count", "number", ["number"]], - ["sqlite3_db_filename", "string", ["number", "string"]], - ["sqlite3_errmsg", "string", ["number"]], - ["sqlite3_exec", "number", ["number", "string", "number", "number", "number"]], - ["sqlite3_extended_result_codes", "number", ["number", "number"]], - ["sqlite3_finalize", "number", ["number"]], - ["sqlite3_interrupt", "void", ["number"]], - ["sqlite3_libversion", "string", []], - ["sqlite3_open", "number", ["string", "number"]], - ["sqlite3_open_v2", "number", ["string", "number", "number", "string"]], - /* sqlite3_prepare_v2() is handled separately due to us requiring two - different sets of semantics for that function. */ - ["sqlite3_reset", "number", ["number"]], - ["sqlite3_result_blob",null,["number", "number", "number", "number"]], - ["sqlite3_result_double",null,["number", "number"]], - ["sqlite3_result_error",null,["number", "string", "number"]], - ["sqlite3_result_int",null,["number", "number"]], - ["sqlite3_result_null",null,["number"]], - ["sqlite3_result_text",null,["number", "string", "number", "number"]], - ["sqlite3_sourceid", "string", []], - ["sqlite3_sql", "string", ["number"]], - ["sqlite3_step", "number", ["number"]], - ["sqlite3_value_blob", "number", ["number"]], - ["sqlite3_value_bytes","number",["number"]], - ["sqlite3_value_double","number",["number"]], - ["sqlite3_value_text", "string", ["number"]], - ["sqlite3_value_type", "number", ["number"]] - //["sqlite3_normalized_sql", "string", ["number"]] - ].forEach((a)=>api[a[0]] = SQM.cwrap.apply(this, a)); - - /** - Proxies for variants of sqlite3_prepare_v2() which have - differing JS/WASM binding semantics. - */ - const prepareMethods = { - /** - This binding expects a JS string as its 2nd argument and - null as its final argument. In order to compile multiple - statements from a single string, the "full" impl (see - below) must be used. - */ - basic: SQM.cwrap('sqlite3_prepare_v2', - "number", ["number", "string", "number"/*MUST always be negative*/, - "number", "number"/*MUST be 0 or null or undefined!*/]), - /* Impl which requires that the 2nd argument be a pointer to - the SQL string, instead of being converted to a - string. This variant is necessary for cases where we - require a non-NULL value for the final argument - (exec()'ing multiple statements from one input - string). For simpler cases, where only the first statement - in the SQL string is required, the wrapper named - sqlite3_prepare_v2() is sufficient and easier to use - because it doesn't require dealing with pointers. - - TODO: hide both of these methods behind a single hand-written - sqlite3_prepare_v2() wrapper which dispatches to the appropriate impl. - */ - full: SQM.cwrap('sqlite3_prepare_v2', - "number", ["number", "number", "number"/*MUST always be negative*/, - "number", "number"]), - }; - - /* Import C-level constants... */ - //console.log("wasmEnum=",SQM.ccall('sqlite3_wasm_enum_json', 'string', [])); - const wasmEnum = JSON.parse(SQM.ccall('sqlite3_wasm_enum_json', 'string', [])); - ['blobFinalizers', 'dataTypes','encodings', - 'openFlags', 'resultCodes','udfFlags' - ].forEach(function(t){ - Object.keys(wasmEnum[t]).forEach(function(k){ - api[k] = wasmEnum[t][k]; - }); - }); - - const utf8Decoder = new TextDecoder('utf-8'); - const typedArrayToString = (str)=>utf8Decoder.decode(str); - //const stringToUint8 = (sql)=>new TextEncoder('utf-8').encode(sql); - - /* Documented inline in the api object. */ - api.sqlite3_prepare_v2 = function(pDb, sql, sqlLen, ppStmt, pzTail){ - if(isSQLableTypedArray(sql)) sql = typedArrayToString(sql); - switch(typeof sql){ - case 'string': return prepareMethods.basic(pDb, sql, -1, ppStmt, null); - case 'number': return prepareMethods.full(pDb, sql, -1, ppStmt, pzTail); - default: toss("Invalid SQL argument type for sqlite3_prepare_v2()."); - } - }; - - /** - Populate api.wasm with several members of the module object. Some of these - will be required by higher-level code. At a minimum: - - getValue(), setValue(), stackSave(), stackRestore(), stackAlloc(), _malloc(), - _free(), addFunction(), removeFunction() - - The rest are exposed primarily for internal use in this API but may well - be useful from higher-level client code. - - All of the functions injected here are part of the - Emscripten-exposed APIs and are documented "elsewhere". Some - are documented in the Emscripten-generated `sqlite3.js` and - some are documented (if at all) in places unknown, possibly - even inaccessible, to us. - */ - [ - // Memory management: - 'getValue','setValue', 'stackSave', 'stackRestore', 'stackAlloc', - 'allocateUTF8OnStack', '_malloc', '_free', - // String utilities: - 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', - // The obligatory "misc" category: - 'addFunction', 'removeFunction' - ].forEach(function(m){ - if(undefined === (api.wasm[m] = SQM[m])){ - toss("Internal init error: Module."+m+" not found."); - } - }); - - /* What follows is colloquially known as "OO API #1". It is a - binding of the sqlite3 API which is designed to be run within - the same thread (main or worker) as the one in which the - sqlite3 WASM binding was initialized. This wrapper cannot use - the sqlite3 binding if, e.g., the wrapper is in the main thread - and the sqlite3 API is in a worker. */ - - /** - The DB class wraps a sqlite3 db handle. - - It accepts the following argument signatures: - - - () - - (undefined) (same effect as ()) - - (filename[,buffer]) - - (buffer) - - Where a buffer indicates a Uint8Array holding an sqlite3 db - image. - - If the filename is provided, only the last component of the - path is used - any path prefix is stripped and certain - "special" characters are replaced with `_`. If no name is - provided, a random name is generated. The resulting filename is - the one used for accessing the db file within root directory of - the emscripten-supplied virtual filesystem, and is set (with no - path part) as the DB object's `filename` property. - - Note that the special sqlite3 db names ":memory:" and "" - (temporary db) have no special meanings here. We can apparently - only export images of DBs which are stored in the - pseudo-filesystem provided by the JS APIs. Since exporting and - importing images is an important usability feature for this - class, ":memory:" DBs are not supported (until/unless we can - find a way to export those as well). The naming semantics will - certainly evolve as this API does. - - The db is opened with a fixed set of flags: - (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXRESCODE). This API may change in the future - permit the caller to provide those flags via an additional - argument. - */ - const DB = function(arg){ - let buffer, fn; - if(arg instanceof Uint8Array){ - buffer = arg; - arg = undefined; - }else if(arguments.length){ /*(filename[,buffer])*/ - if('string'===typeof arg){ - const p = arg.split('/').pop().replace(':','_'); - if(p) fn = p; - if(arguments.length>1){ - buffer = arguments[1]; - } - }else if(undefined!==arg){ - toss("Invalid arguments to DB constructor.", - "Expecting (), (undefined), (name,buffer),", - "or (buffer), where buffer an sqlite3 db ", - "as a Uint8Array."); - } - } - if(!fn){ - fn = "db-"+((Math.random() * 10000000) | 0)+ - "-"+((Math.random() * 10000000) | 0)+".sqlite3"; - } - if(buffer){ - if(!(buffer instanceof Uint8Array)){ - toss("Expecting Uint8Array image of db contents."); - } - FS.createDataFile("/", fn, buffer, true, true); - } - const stack = api.wasm.stackSave(); - const ppDb = api.wasm.stackAlloc(4) /* output (sqlite3**) arg */; - api.wasm.setValue(ppDb, 0, "i32"); - try { - this.checkRc(api.sqlite3_open_v2(fn, ppDb, api.SQLITE_OPEN_READWRITE - | api.SQLITE_OPEN_CREATE - | api.SQLITE_OPEN_EXRESCODE, null)); - } - finally{api.wasm.stackRestore(stack);} - this._pDb = api.wasm.getValue(ppDb, "i32"); - this.filename = fn; - this._statements = {/*map of open Stmt _pointers_ to Stmt*/}; - this._udfs = {/*map of UDF names to wasm function _pointers_*/}; - }; - - /** - Internal-use enum for mapping JS types to DB-bindable types. - These do not (and need not) line up with the SQLITE_type - values. All values in this enum must be truthy and distinct - but they need not be numbers. - */ - const BindTypes = { - null: 1, - number: 2, - string: 3, - boolean: 4, - blob: 5 - }; - BindTypes['undefined'] == BindTypes.null; - - /** - This class wraps sqlite3_stmt. Calling this constructor - directly will trigger an exception. Use DB.prepare() to create - new instances. - */ - const Stmt = function(){ - if(BindTypes!==arguments[2]){ - toss("Do not call the Stmt constructor directly. Use DB.prepare()."); - } - this.db = arguments[0]; - this._pStmt = arguments[1]; - this.columnCount = api.sqlite3_column_count(this._pStmt); - this.parameterCount = api.sqlite3_bind_parameter_count(this._pStmt); - }; - - /** Throws if the given DB has been closed, else it is returned. */ - const affirmDbOpen = function(db){ - if(!db._pDb) toss("DB has been closed."); - return db; - }; - - /** - Expects to be passed (arguments) from DB.exec() and - DB.execMulti(). Does the argument processing/validation, throws - on error, and returns a new object on success: - - { sql: the SQL, opt: optionsObj, cbArg: function} - - cbArg is only set if the opt.callback is set, in which case - it's a function which expects to be passed the current Stmt - and returns the callback argument of the type indicated by - the input arguments. - */ - const parseExecArgs = function(args){ - const out = {opt:{}}; - switch(args.length){ - case 1: - if('string'===typeof args[0] || isSQLableTypedArray(args[0])){ - out.sql = args[0]; - }else if(args[0] && 'object'===typeof args[0]){ - out.opt = args[0]; - out.sql = out.opt.sql; - } - break; - case 2: - out.sql = args[0]; - out.opt = args[1]; - break; - default: toss("Invalid argument count for exec()."); - }; - if(isSQLableTypedArray(out.sql)){ - out.sql = typedArrayToString(out.sql); - }else if(Array.isArray(out.sql)){ - out.sql = out.sql.join(''); - }else if('string'!==typeof out.sql){ - toss("Missing SQL argument."); - } - if(out.opt.callback || out.opt.resultRows){ - switch((undefined===out.opt.rowMode) - ? 'stmt' : out.opt.rowMode) { - case 'object': out.cbArg = (stmt)=>stmt.get({}); break; - case 'array': out.cbArg = (stmt)=>stmt.get([]); break; - case 'stmt': out.cbArg = (stmt)=>stmt; break; - default: toss("Invalid rowMode:",out.opt.rowMode); - } - } - return out; - }; - - /** If object opts has _its own_ property named p then that - property's value is returned, else dflt is returned. */ - const getOwnOption = (opts, p, dflt)=> - opts.hasOwnProperty(p) ? opts[p] : dflt; - - DB.prototype = { - /** - Expects to be given an sqlite3 API result code. If it is - falsy, this function returns this object, else it throws an - exception with an error message from sqlite3_errmsg(), - using this object's db handle. Note that if it's passed a - non-error code like SQLITE_ROW or SQLITE_DONE, it will - still throw but the error string might be "Not an error." - The various non-0 non-error codes need to be checked for in - client code where they are expected. - */ - checkRc: function(sqliteResultCode){ - if(!sqliteResultCode) return this; - toss("sqlite result code",sqliteResultCode+":", - api.sqlite3_errmsg(this._pDb) || "Unknown db error."); - }, - /** - Finalizes all open statements and closes this database - connection. This is a no-op if the db has already been - closed. If the db is open and alsoUnlink is truthy then the - this.filename entry in the pseudo-filesystem will also be - removed (and any error in that attempt is silently - ignored). - */ - close: function(alsoUnlink){ - if(this._pDb){ - let s; - const that = this; - Object.keys(this._statements).forEach(function(k,s){ - delete that._statements[k]; - if(s && s._pStmt) s.finalize(); - }); - Object.values(this._udfs).forEach(api.wasm.removeFunction); - delete this._udfs; - delete this._statements; - api.sqlite3_close_v2(this._pDb); - delete this._pDb; - if(this.filename){ - if(alsoUnlink){ - try{SQM.FS.unlink('/'+this.filename);} - catch(e){/*ignored*/} - } - delete this.filename; - } - } - }, - /** - Similar to this.filename but will return NULL for - special names like ":memory:". Not of much use until - we have filesystem support. Throws if the DB has - been closed. If passed an argument it then it will return - the filename of the ATTACHEd db with that name, else it assumes - a name of `main`. - */ - fileName: function(dbName){ - return api.sqlite3_db_filename(affirmDbOpen(this)._pDb, dbName||"main"); - }, - /** - Compiles the given SQL and returns a prepared Stmt. This is - the only way to create new Stmt objects. Throws on error. - - The given SQL must be a string, a Uint8Array holding SQL, - or a WASM pointer to memory allocated using - api.wasm.allocateUTF8OnStack() (or equivalent (a term which - is yet to be defined precisely)). - */ - prepare: function(sql){ - affirmDbOpen(this); - const stack = api.wasm.stackSave(); - const ppStmt = api.wasm.stackAlloc(4)/* output (sqlite3_stmt**) arg */; - api.wasm.setValue(ppStmt, 0, "i32"); - try {this.checkRc(api.sqlite3_prepare_v2(this._pDb, sql, -1, ppStmt, null));} - finally {api.wasm.stackRestore(stack);} - const pStmt = api.wasm.getValue(ppStmt, "i32"); - if(!pStmt) toss("Cannot prepare empty SQL."); - const stmt = new Stmt(this, pStmt, BindTypes); - this._statements[pStmt] = stmt; - return stmt; - }, - /** - This function works like execMulti(), and takes most of the - same arguments, but is more efficient (performs much less - work) when the input SQL is only a single statement. If - passed a multi-statement SQL, it only processes the first - one. - - This function supports the following additional options not - supported by execMulti(): - - - .multi: if true, this function acts as a proxy for - execMulti() and behaves identically to that function. - - - .resultRows: if this is an array, each row of the result - set (if any) is appended to it in the format specified - for the `rowMode` property, with the exception that the - only legal values for `rowMode` in this case are 'array' - or 'object', neither of which is the default. It is legal - to use both `resultRows` and `callback`, but `resultRows` - is likely much simpler to use for small data sets and can - be used over a WebWorker-style message interface. - - - .columnNames: if this is an array and the query has - result columns, the array is passed to - Stmt.getColumnNames() to append the column names to it - (regardless of whether the query produces any result - rows). If the query has no result columns, this value is - unchanged. - - The following options to execMulti() are _not_ supported by - this method (they are simply ignored): - - - .saveSql - */ - exec: function(/*(sql [,optionsObj]) or (optionsObj)*/){ - affirmDbOpen(this); - const arg = parseExecArgs(arguments); - if(!arg.sql) return this; - else if(arg.opt.multi){ - return this.execMulti(arg, undefined, BindTypes); - } - const opt = arg.opt; - let stmt, rowTarget; - try { - if(Array.isArray(opt.resultRows)){ - if(opt.rowMode!=='array' && opt.rowMode!=='object'){ - toss("Invalid rowMode for resultRows array: must", - "be one of 'array' or 'object'."); - } - rowTarget = opt.resultRows; - } - stmt = this.prepare(arg.sql); - if(stmt.columnCount && Array.isArray(opt.columnNames)){ - stmt.getColumnNames(opt.columnNames); - } - if(opt.bind) stmt.bind(opt.bind); - if(opt.callback || rowTarget){ - while(stmt.step()){ - const row = arg.cbArg(stmt); - if(rowTarget) rowTarget.push(row); - if(opt.callback){ - stmt._isLocked = true; - opt.callback(row, stmt); - stmt._isLocked = false; - } - } - }else{ - stmt.step(); - } - }finally{ - if(stmt){ - delete stmt._isLocked; - stmt.finalize(); - } - } - return this; - - }/*exec()*/, - /** - Executes one or more SQL statements in the form of a single - string. Its arguments must be either (sql,optionsObject) or - (optionsObject). In the latter case, optionsObject.sql - must contain the SQL to execute. Returns this - object. Throws on error. - - If no SQL is provided, or a non-string is provided, an - exception is triggered. Empty SQL, on the other hand, is - simply a no-op. - - The optional options object may contain any of the following - properties: - - - .sql = the SQL to run (unless it's provided as the first - argument). This must be of type string, Uint8Array, or an - array of strings (in which case they're concatenated - together as-is, with no separator between elements, - before evaluation). - - - .bind = a single value valid as an argument for - Stmt.bind(). This is ONLY applied to the FIRST non-empty - statement in the SQL which has any bindable - parameters. (Empty statements are skipped entirely.) - - - .callback = a function which gets called for each row of - the FIRST statement in the SQL which has result - _columns_, but only if that statement has any result - _rows_. The second argument passed to the callback is - always the current Stmt object (so that the caller may - collect column names, or similar). The first argument - passed to the callback defaults to the current Stmt - object but may be changed with ... - - - .rowMode = a string describing what type of argument - should be passed as the first argument to the callback. A - value of 'object' causes the results of `stmt.get({})` to - be passed to the object. A value of 'array' causes the - results of `stmt.get([])` to be passed to the callback. - A value of 'stmt' is equivalent to the default, passing - the current Stmt to the callback (noting that it's always - passed as the 2nd argument). Any other value triggers an - exception. - - - saveSql = an optional array. If set, the SQL of each - executed statement is appended to this array before the - statement is executed (but after it is prepared - we - don't have the string until after that). Empty SQL - statements are elided. - - See also the exec() method, which is a close cousin of this - one. - - ACHTUNG #1: The callback MUST NOT modify the Stmt - object. Calling any of the Stmt.get() variants, - Stmt.getColumnName(), or similar, is legal, but calling - step() or finalize() is not. Routines which are illegal - in this context will trigger an exception. - - ACHTUNG #2: The semantics of the `bind` and `callback` - options may well change or those options may be removed - altogether for this function (but retained for exec()). - Generally speaking, neither bind parameters nor a callback - are generically useful when executing multi-statement SQL. - */ - execMulti: function(/*(sql [,obj]) || (obj)*/){ - affirmDbOpen(this); - const arg = (BindTypes===arguments[2] - /* ^^^ Being passed on from exec() */ - ? arguments[0] : parseExecArgs(arguments)); - if(!arg.sql) return this; - const opt = arg.opt; - const stack = api.wasm.stackSave(); - let stmt; - let bind = opt.bind; - let rowMode = ( - (opt.callback && opt.rowMode) - ? opt.rowMode : false); - try{ - const sql = isSQLableTypedArray(arg.sql) - ? typedArrayToString(arg.sql) - : arg.sql; - let pSql = api.wasm.allocateUTF8OnStack(sql) - const ppStmt = api.wasm.stackAlloc(8) /* output (sqlite3_stmt**) arg */; - const pzTail = ppStmt + 4 /* final arg to sqlite3_prepare_v2_sqlptr() */; - while(api.wasm.getValue(pSql, "i8")){ - api.wasm.setValue(ppStmt, 0, "i32"); - api.wasm.setValue(pzTail, 0, "i32"); - this.checkRc(api.sqlite3_prepare_v2( - this._pDb, pSql, -1, ppStmt, pzTail - )); - const pStmt = api.wasm.getValue(ppStmt, "i32"); - pSql = api.wasm.getValue(pzTail, "i32"); - if(!pStmt) continue; - if(opt.saveSql){ - opt.saveSql.push(api.sqlite3_sql(pStmt).trim()); - } - stmt = new Stmt(this, pStmt, BindTypes); - if(bind && stmt.parameterCount){ - stmt.bind(bind); - bind = null; - } - if(opt.callback && null!==rowMode && stmt.columnCount){ - while(stmt.step()){ - stmt._isLocked = true; - callback(arg.cbArg(stmt), stmt); - stmt._isLocked = false; - } - rowMode = null; - }else{ - // Do we need to while(stmt.step()){} here? - stmt.step(); - } - stmt.finalize(); - stmt = null; - } - }finally{ - if(stmt){ - delete stmt._isLocked; - stmt.finalize(); - } - api.wasm.stackRestore(stack); - } - return this; - }/*execMulti()*/, - /** - Creates a new scalar UDF (User-Defined Function) which is - accessible via SQL code. This function may be called in any - of the following forms: - - - (name, function) - - (name, function, optionsObject) - - (name, optionsObject) - - (optionsObject) - - In the final two cases, the function must be defined as the - 'callback' property of the options object. In the final - case, the function's name must be the 'name' property. - - This can only be used to create scalar functions, not - aggregate or window functions. UDFs cannot be removed from - a DB handle after they're added. - - On success, returns this object. Throws on error. - - When called from SQL, arguments to the UDF, and its result, - will be converted between JS and SQL with as much fidelity - as is feasible, triggering an exception if a type - conversion cannot be determined. Some freedom is afforded - to numeric conversions due to friction between the JS and C - worlds: integers which are larger than 32 bits will be - treated as doubles, as JS does not support 64-bit integers - and it is (as of this writing) illegal to use WASM - functions which take or return 64-bit integers from JS. - - The optional options object may contain flags to modify how - the function is defined: - - - .arity: the number of arguments which SQL calls to this - function expect or require. The default value is the - callback's length property (i.e. the number of declared - parameters it has). A value of -1 means that the function - is variadic and may accept any number of arguments, up to - sqlite3's compile-time limits. sqlite3 will enforce the - argument count if is zero or greater. - - The following properties correspond to flags documented at: - - https://sqlite.org/c3ref/create_function.html - - - .deterministic = SQLITE_DETERMINISTIC - - .directOnly = SQLITE_DIRECTONLY - - .innocuous = SQLITE_INNOCUOUS - - Maintenance reminder: the ability to add new - WASM-accessible functions to the runtime requires that the - WASM build is compiled with emcc's `-sALLOW_TABLE_GROWTH` - flag. - */ - createFunction: function f(name, callback,opt){ - switch(arguments.length){ - case 1: /* (optionsObject) */ - opt = name; - name = opt.name; - callback = opt.callback; - break; - case 2: /* (name, callback|optionsObject) */ - if(!(callback instanceof Function)){ - opt = callback; - callback = opt.callback; - } - break; - default: break; - } - if(!opt) opt = {}; - if(!(callback instanceof Function)){ - toss("Invalid arguments: expecting a callback function."); - }else if('string' !== typeof name){ - toss("Invalid arguments: missing function name."); - } - if(!f._extractArgs){ - /* Static init */ - f._extractArgs = function(argc, pArgv){ - let i, pVal, valType, arg; - const tgt = []; - for(i = 0; i < argc; ++i){ - pVal = api.wasm.getValue(pArgv + (4 * i), "i32"); - valType = api.sqlite3_value_type(pVal); - switch(valType){ - case api.SQLITE_INTEGER: - case api.SQLITE_FLOAT: - arg = api.sqlite3_value_double(pVal); - break; - case api.SQLITE_TEXT: - arg = api.sqlite3_value_text(pVal); - break; - case api.SQLITE_BLOB:{ - const n = api.sqlite3_value_bytes(pVal); - const pBlob = api.sqlite3_value_blob(pVal); - arg = new Uint8Array(n); - let i; - const heap = n ? api.wasm.HEAP8() : false; - for(i = 0; i < n; ++i) arg[i] = heap[pBlob+i]; - break; - } - default: - arg = null; break; - } - tgt.push(arg); - } - return tgt; - }/*_extractArgs()*/; - f._setResult = function(pCx, val){ - switch(typeof val) { - case 'boolean': - api.sqlite3_result_int(pCx, val ? 1 : 0); - break; - case 'number': { - (isInt32(val) - ? api.sqlite3_result_int - : api.sqlite3_result_double)(pCx, val); - break; - } - case 'string': - api.sqlite3_result_text(pCx, val, -1, api.SQLITE_TRANSIENT); - break; - case 'object': - if(null===val) { - api.sqlite3_result_null(pCx); - break; - }else if(isBindableTypedArray(val)){ - const pBlob = api.wasm.mallocFromTypedArray(val); - api.sqlite3_result_blob(pCx, pBlob, val.byteLength, - api.SQLITE_TRANSIENT); - api.wasm._free(pBlob); - break; - } - // else fall through - default: - toss("Don't not how to handle this UDF result value:",val); - }; - }/*_setResult()*/; - }/*static init*/ - const wrapper = function(pCx, argc, pArgv){ - try{ - f._setResult(pCx, callback.apply(null, f._extractArgs(argc, pArgv))); - }catch(e){ - api.sqlite3_result_error(pCx, e.message, -1); - } - }; - const pUdf = api.wasm.addFunction(wrapper, "viii"); - let fFlags = 0; - if(getOwnOption(opt, 'deterministic')) fFlags |= api.SQLITE_DETERMINISTIC; - if(getOwnOption(opt, 'directOnly')) fFlags |= api.SQLITE_DIRECTONLY; - if(getOwnOption(opt, 'innocuous')) fFlags |= api.SQLITE_INNOCUOUS; - name = name.toLowerCase(); - try { - this.checkRc(api.sqlite3_create_function_v2( - this._pDb, name, - (opt.hasOwnProperty('arity') ? +opt.arity : callback.length), - api.SQLITE_UTF8 | fFlags, null/*pApp*/, pUdf, - null/*xStep*/, null/*xFinal*/, null/*xDestroy*/)); - }catch(e){ - api.wasm.removeFunction(pUdf); - throw e; - } - if(this._udfs.hasOwnProperty(name)){ - api.wasm.removeFunction(this._udfs[name]); - } - this._udfs[name] = pUdf; - return this; - }/*createFunction()*/, - /** - Prepares the given SQL, step()s it one time, and returns - the value of the first result column. If it has no results, - undefined is returned. If passed a second argument, it is - treated like an argument to Stmt.bind(), so may be any type - supported by that function. Throws on error (e.g. malformed - SQL). - */ - selectValue: function(sql,bind){ - let stmt, rc; - try { - stmt = this.prepare(sql).bind(bind); - if(stmt.step()) rc = stmt.get(0); - }finally{ - if(stmt) stmt.finalize(); - } - return rc; - }, - - /** - Exports a copy of this db's file as a Uint8Array and - returns it. It is technically not legal to call this while - any prepared statement are currently active because, - depending on the platform, it might not be legal to read - the db while a statement is locking it. Throws if this db - is not open or has any opened statements. - - The resulting buffer can be passed to this class's - constructor to restore the DB. - - Maintenance reminder: the corresponding sql.js impl of this - feature closes the current db, finalizing any active - statements and (seemingly unnecessarily) destroys any UDFs, - copies the file, and then re-opens it (without restoring - the UDFs). Those gymnastics are not necessary on the tested - platform but might be necessary on others. Because of that - eventuality, this interface currently enforces that no - statements are active when this is run. It will throw if - any are. - */ - exportBinaryImage: function(){ - affirmDbOpen(this); - if(Object.keys(this._statements).length){ - toss("Cannot export with prepared statements active!", - "finalize() all statements and try again."); - } - return FS.readFile(this.filename, {encoding:"binary"}); - } - }/*DB.prototype*/; - - - /** Throws if the given Stmt has been finalized, else stmt is - returned. */ - const affirmStmtOpen = function(stmt){ - if(!stmt._pStmt) toss("Stmt has been closed."); - return stmt; - }; - - /** Returns an opaque truthy value from the BindTypes - enum if v's type is a valid bindable type, else - returns a falsy value. As a special case, a value of - undefined is treated as a bind type of null. */ - const isSupportedBindType = function(v){ - let t = BindTypes[(null===v||undefined===v) ? 'null' : typeof v]; - switch(t){ - case BindTypes.boolean: - case BindTypes.null: - case BindTypes.number: - case BindTypes.string: - return t; - default: - //console.log("isSupportedBindType",t,v); - return isBindableTypedArray(v) ? BindTypes.blob : undefined; - } - }; - - /** - If isSupportedBindType(v) returns a truthy value, this - function returns that value, else it throws. - */ - const affirmSupportedBindType = function(v){ - //console.log('affirmSupportedBindType',v); - return isSupportedBindType(v) || toss("Unsupported bind() argument type:",typeof v); - }; - - /** - If key is a number and within range of stmt's bound parameter - count, key is returned. - - If key is not a number then it is checked against named - parameters. If a match is found, its index is returned. - - Else it throws. - */ - const affirmParamIndex = function(stmt,key){ - const n = ('number'===typeof key) - ? key : api.sqlite3_bind_parameter_index(stmt._pStmt, key); - if(0===n || !isInt32(n)){ - toss("Invalid bind() parameter name: "+key); - } - else if(n<1 || n>stmt.parameterCount) toss("Bind index",key,"is out of range."); - return n; - }; - - /** Throws if ndx is not an integer or if it is out of range - for stmt.columnCount, else returns stmt. - - Reminder: this will also fail after the statement is finalized - but the resulting error will be about an out-of-bounds column - index. - */ - const affirmColIndex = function(stmt,ndx){ - if((ndx !== (ndx|0)) || ndx<0 || ndx>=stmt.columnCount){ - toss("Column index",ndx,"is out of range."); - } - return stmt; - }; - - /** - If stmt._isLocked is truthy, this throws an exception - complaining that the 2nd argument (an operation name, - e.g. "bind()") is not legal while the statement is "locked". - Locking happens before an exec()-like callback is passed a - statement, to ensure that the callback does not mutate or - finalize the statement. If it does not throw, it returns stmt. - */ - const affirmUnlocked = function(stmt,currentOpName){ - if(stmt._isLocked){ - toss("Operation is illegal when statement is locked:",currentOpName); - } - return stmt; - }; - - /** - Binds a single bound parameter value on the given stmt at the - given index (numeric or named) using the given bindType (see - the BindTypes enum) and value. Throws on error. Returns stmt on - success. - */ - const bindOne = function f(stmt,ndx,bindType,val){ - affirmUnlocked(stmt, 'bind()'); - if(!f._){ - f._ = { - string: function(stmt, ndx, val, asBlob){ - if(1){ - /* _Hypothetically_ more efficient than the impl in the 'else' block. */ - const stack = api.wasm.stackSave(); - try{ - const n = api.wasm.lengthBytesUTF8(val)+1/*required for NUL terminator*/; - const pStr = api.wasm.stackAlloc(n); - api.wasm.stringToUTF8Array(val, api.wasm.HEAPU8(), pStr, n); - const f = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; - return f(stmt._pStmt, ndx, pStr, n-1, api.SQLITE_TRANSIENT); - }finally{ - api.wasm.stackRestore(stack); - } - }else{ - const bytes = api.wasm.intArrayFromString(val,true); - const pStr = api.wasm._malloc(bytes.length || 1); - api.wasm.HEAPU8().set(bytes.length ? bytes : [0], pStr); - try{ - const f = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; - return f(stmt._pStmt, ndx, pStr, bytes.length, api.SQLITE_TRANSIENT); - }finally{ - api.wasm._free(pStr); - } - } - } - }; - } - affirmSupportedBindType(val); - ndx = affirmParamIndex(stmt,ndx); - let rc = 0; - switch((null===val || undefined===val) ? BindTypes.null : bindType){ - case BindTypes.null: - rc = api.sqlite3_bind_null(stmt._pStmt, ndx); - break; - case BindTypes.string:{ - rc = f._.string(stmt, ndx, val, false); - break; - } - case BindTypes.number: { - const m = (isInt32(val) - ? api.sqlite3_bind_int - /*It's illegal to bind a 64-bit int - from here*/ - : api.sqlite3_bind_double); - rc = m(stmt._pStmt, ndx, val); - break; - } - case BindTypes.boolean: - rc = api.sqlite3_bind_int(stmt._pStmt, ndx, val ? 1 : 0); - break; - case BindTypes.blob: { - if('string'===typeof val){ - rc = f._.string(stmt, ndx, val, true); - }else if(!isBindableTypedArray(val)){ - toss("Binding a value as a blob requires", - "that it be a string, Uint8Array, or Int8Array."); - }else if(1){ - /* _Hypothetically_ more efficient than the impl in the 'else' block. */ - const stack = api.wasm.stackSave(); - try{ - const pBlob = api.wasm.stackAlloc(val.byteLength || 1); - api.wasm.HEAP8().set(val.byteLength ? val : [0], pBlob) - rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, val.byteLength, - api.SQLITE_TRANSIENT); - }finally{ - api.wasm.stackRestore(stack); - } - }else{ - const pBlob = api.wasm.mallocFromTypedArray(val); - try{ - rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, val.byteLength, - api.SQLITE_TRANSIENT); - }finally{ - api.wasm._free(pBlob); - } - } - break; - } - default: - console.warn("Unsupported bind() argument type:",val); - toss("Unsupported bind() argument type."); - } - if(rc) stmt.db.checkRc(rc); - return stmt; - }; - - Stmt.prototype = { - /** - "Finalizes" this statement. This is a no-op if the - statement has already been finalizes. Returns - undefined. Most methods in this class will throw if called - after this is. - */ - finalize: function(){ - if(this._pStmt){ - affirmUnlocked(this,'finalize()'); - delete this.db._statements[this._pStmt]; - api.sqlite3_finalize(this._pStmt); - delete this.columnCount; - delete this.parameterCount; - delete this._pStmt; - delete this.db; - delete this._isLocked; - } - }, - /** Clears all bound values. Returns this object. - Throws if this statement has been finalized. */ - clearBindings: function(){ - affirmUnlocked(affirmStmtOpen(this), 'clearBindings()') - api.sqlite3_clear_bindings(this._pStmt); - this._mayGet = false; - return this; - }, - /** - Resets this statement so that it may be step()ed again - from the beginning. Returns this object. Throws if this - statement has been finalized. - - If passed a truthy argument then this.clearBindings() is - also called, otherwise any existing bindings, along with - any memory allocated for them, are retained. - */ - reset: function(alsoClearBinds){ - affirmUnlocked(this,'reset()'); - if(alsoClearBinds) this.clearBindings(); - api.sqlite3_reset(affirmStmtOpen(this)._pStmt); - this._mayGet = false; - return this; - }, - /** - Binds one or more values to its bindable parameters. It - accepts 1 or 2 arguments: - - If passed a single argument, it must be either an array, an - object, or a value of a bindable type (see below). - - If passed 2 arguments, the first one is the 1-based bind - index or bindable parameter name and the second one must be - a value of a bindable type. - - Bindable value types: - - - null is bound as NULL. - - - undefined as a standalone value is a no-op intended to - simplify certain client-side use cases: passing undefined - as a value to this function will not actually bind - anything and this function will skip confirmation that - binding is even legal. (Those semantics simplify certain - client-side uses.) Conversely, a value of undefined as an - array or object property when binding an array/object - (see below) is treated the same as null. - - - Numbers are bound as either doubles or integers: doubles - if they are larger than 32 bits, else double or int32, - depending on whether they have a fractional part. (It is, - as of this writing, illegal to call (from JS) a WASM - function which either takes or returns an int64.) - Booleans are bound as integer 0 or 1. It is not expected - the distinction of binding doubles which have no - fractional parts is integers is significant for the - majority of clients due to sqlite3's data typing - model. This API does not currently support the BigInt - type. - - - Strings are bound as strings (use bindAsBlob() to force - blob binding). - - - Uint8Array and Int8Array instances are bound as blobs. - (TODO: support binding other TypedArray types with larger - int sizes.) - - If passed an array, each element of the array is bound at - the parameter index equal to the array index plus 1 - (because arrays are 0-based but binding is 1-based). - - If passed an object, each object key is treated as a - bindable parameter name. The object keys _must_ match any - bindable parameter names, including any `$`, `@`, or `:` - prefix. Because `$` is a legal identifier chararacter in - JavaScript, that is the suggested prefix for bindable - parameters: `stmt.bind({$a: 1, $b: 2})`. - - It returns this object on success and throws on - error. Errors include: - - - Any bind index is out of range, a named bind parameter - does not match, or this statement has no bindable - parameters. - - - Any value to bind is of an unsupported type. - - - Passed no arguments or more than two. - - - The statement has been finalized. - */ - bind: function(/*[ndx,] arg*/){ - affirmStmtOpen(this); - let ndx, arg; - switch(arguments.length){ - case 1: ndx = 1; arg = arguments[0]; break; - case 2: ndx = arguments[0]; arg = arguments[1]; break; - default: toss("Invalid bind() arguments."); - } - if(undefined===arg){ - /* It might seem intuitive to bind undefined as NULL - but this approach simplifies certain client-side - uses when passing on arguments between 2+ levels of - functions. */ - return this; - }else if(!this.parameterCount){ - toss("This statement has no bindable parameters."); - } - this._mayGet = false; - if(null===arg){ - /* bind NULL */ - return bindOne(this, ndx, BindTypes.null, arg); - } - else if(Array.isArray(arg)){ - /* bind each entry by index */ - if(1!==arguments.length){ - toss("When binding an array, an index argument is not permitted."); - } - arg.forEach((v,i)=>bindOne(this, i+1, affirmSupportedBindType(v), v)); - return this; - } - else if('object'===typeof arg/*null was checked above*/ - && !isBindableTypedArray(arg)){ - /* Treat each property of arg as a named bound parameter. */ - if(1!==arguments.length){ - toss("When binding an object, an index argument is not permitted."); - } - Object.keys(arg) - .forEach(k=>bindOne(this, k, - affirmSupportedBindType(arg[k]), - arg[k])); - return this; - }else{ - return bindOne(this, ndx, affirmSupportedBindType(arg), arg); - } - toss("Should not reach this point."); - }, - /** - Special case of bind() which binds the given value using - the BLOB binding mechanism instead of the default selected - one for the value. The ndx may be a numbered or named bind - index. The value must be of type string, null/undefined - (both treated as null), or a TypedArray of a type supported - by the bind() API. - - If passed a single argument, a bind index of 1 is assumed. - */ - bindAsBlob: function(ndx,arg){ - affirmStmtOpen(this); - if(1===arguments.length){ - arg = ndx; - ndx = 1; - } - const t = affirmSupportedBindType(arg); - if(BindTypes.string !== t && BindTypes.blob !== t - && BindTypes.null !== t){ - toss("Invalid value type for bindAsBlob()"); - } - this._mayGet = false; - return bindOne(this, ndx, BindTypes.blob, arg); - }, - /** - Steps the statement one time. If the result indicates that - a row of data is available, true is returned. If no row of - data is available, false is returned. Throws on error. - */ - step: function(){ - affirmUnlocked(this, 'step()'); - const rc = api.sqlite3_step(affirmStmtOpen(this)._pStmt); - switch(rc){ - case api.SQLITE_DONE: return this._mayGet = false; - case api.SQLITE_ROW: return this._mayGet = true; - default: - this._mayGet = false; - console.warn("sqlite3_step() rc=",rc,"SQL =", - api.sqlite3_sql(this._pStmt)); - this.db.checkRc(rc); - }; - }, - /** - Fetches the value from the given 0-based column index of - the current data row, throwing if index is out of range. - - Requires that step() has just returned a truthy value, else - an exception is thrown. - - By default it will determine the data type of the result - automatically. If passed a second arugment, it must be one - of the enumeration values for sqlite3 types, which are - defined as members of the sqlite3 module: SQLITE_INTEGER, - SQLITE_FLOAT, SQLITE_TEXT, SQLITE_BLOB. Any other value, - except for undefined, will trigger an exception. Passing - undefined is the same as not passing a value. It is legal - to, e.g., fetch an integer value as a string, in which case - sqlite3 will convert the value to a string. - - If ndx is an array, this function behaves a differently: it - assigns the indexes of the array, from 0 to the number of - result columns, to the values of the corresponding column, - and returns that array. - - If ndx is a plain object, this function behaves even - differentlier: it assigns the properties of the object to - the values of their corresponding result columns. - - Blobs are returned as Uint8Array instances. - - Potential TODO: add type ID SQLITE_JSON, which fetches the - result as a string and passes it (if it's not null) to - JSON.parse(), returning the result of that. Until then, - getJSON() can be used for that. - */ - get: function(ndx,asType){ - if(!affirmStmtOpen(this)._mayGet){ - toss("Stmt.step() has not (recently) returned true."); - } - if(Array.isArray(ndx)){ - let i = 0; - while(i32bits */ - } - case api.SQLITE_FLOAT: - return api.sqlite3_column_double(this._pStmt, ndx); - case api.SQLITE_TEXT: - return api.sqlite3_column_text(this._pStmt, ndx); - case api.SQLITE_BLOB: { - const n = api.sqlite3_column_bytes(this._pStmt, ndx), - ptr = api.sqlite3_column_blob(this._pStmt, ndx), - rc = new Uint8Array(n), - heap = n ? api.wasm.HEAP8() : false; - for(let i = 0; i < n; ++i) rc[i] = heap[ptr + i]; - if(n && this.db._blobXfer instanceof Array){ - /* This is an optimization soley for the - Worker-based API. These values will be - transfered to the main thread directly - instead of being copied. */ - this.db._blobXfer.push(rc.buffer); - } - return rc; - } - default: toss("Don't know how to translate", - "type of result column #"+ndx+"."); - } - abort("Not reached."); - }, - /** Equivalent to get(ndx) but coerces the result to an - integer. */ - getInt: function(ndx){return this.get(ndx,api.SQLITE_INTEGER)}, - /** Equivalent to get(ndx) but coerces the result to a - float. */ - getFloat: function(ndx){return this.get(ndx,api.SQLITE_FLOAT)}, - /** Equivalent to get(ndx) but coerces the result to a - string. */ - getString: function(ndx){return this.get(ndx,api.SQLITE_TEXT)}, - /** Equivalent to get(ndx) but coerces the result to a - Uint8Array. */ - getBlob: function(ndx){return this.get(ndx,api.SQLITE_BLOB)}, - /** - A convenience wrapper around get() which fetches the value - as a string and then, if it is not null, passes it to - JSON.parse(), returning that result. Throws if parsing - fails. If the result is null, null is returned. An empty - string, on the other hand, will trigger an exception. - */ - getJSON: function(ndx){ - const s = this.get(ndx, api.SQLITE_STRING); - return null===s ? s : JSON.parse(s); - }, - /** - Returns the result column name of the given index, or - throws if index is out of bounds or this statement has been - finalized. This can be used without having run step() - first. - */ - getColumnName: function(ndx){ - return api.sqlite3_column_name( - affirmColIndex(affirmStmtOpen(this),ndx)._pStmt, ndx - ); - }, - /** - If this statement potentially has result columns, this - function returns an array of all such names. If passed an - array, it is used as the target and all names are appended - to it. Returns the target array. Throws if this statement - cannot have result columns. This object's columnCount member - holds the number of columns. - */ - getColumnNames: function(tgt){ - affirmColIndex(affirmStmtOpen(this),0); - if(!tgt) tgt = []; - for(let i = 0; i < this.columnCount; ++i){ - tgt.push(api.sqlite3_column_name(this._pStmt, i)); - } - return tgt; - }, - /** - If this statement has named bindable parameters and the - given name matches one, its 1-based bind index is - returned. If no match is found, 0 is returned. If it has no - bindable parameters, the undefined value is returned. - */ - getParamIndex: function(name){ - return (affirmStmtOpen(this).parameterCount - ? api.sqlite3_bind_parameter_index(this._pStmt, name) - : undefined); - } - }/*Stmt.prototype*/; - - /** OO binding's namespace. */ - const SQLite3 = { - version: { - lib: api.sqlite3_libversion(), - ooApi: "0.0.1" - }, - DB, - Stmt, - /** - Reports info about compile-time options. It has several - distinct uses: - - If optName is an array then it is expected to be a list of - compilation options and this function returns an object - which maps each such option to true or false, indicating - whether or not the given option was included in this - build. That object is returned. - - If optName is an object, its keys are expected to be - compilation options and this function sets each entry to - true or false. That object is returned. - - If passed no arguments then it returns an object mapping - all known compilation options to their compile-time values, - or boolean true if they are defined with no value. This - result, which is relatively expensive to compute, is cached - and returned for future no-argument calls. - - In all other cases it returns true if the given option was - active when when compiling the sqlite3 module, else false. - - Compile-time option names may optionally include their - "SQLITE_" prefix. When it returns an object of all options, - the prefix is elided. - */ - compileOptionUsed: function f(optName){ - if(!arguments.length){ - if(f._result) return f._result; - else if(!f._opt){ - f._rx = /^([^=]+)=(.+)/; - f._rxInt = /^-?\d+$/; - f._opt = function(opt, rv){ - const m = f._rx.exec(opt); - rv[0] = (m ? m[1] : opt); - rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true; - }; - } - const rc = {}, ov = [0,0]; - let i = 0, k; - while((k = api.sqlite3_compileoption_get(i++))){ - f._opt(k,ov); - rc[ov[0]] = ov[1]; - } - return f._result = rc; - }else if(Array.isArray(optName)){ - const rc = {}; - optName.forEach((v)=>{ - rc[v] = api.sqlite3_compileoption_used(v); - }); - return rc; - }else if('object' === typeof optName){ - Object.keys(optName).forEach((k)=> { - optName[k] = api.sqlite3_compileoption_used(k); - }); - return optName; - } - return ( - 'string'===typeof optName - ) ? !!api.sqlite3_compileoption_used(optName) : false; - } - }/*SQLite3 object*/; - - namespace.sqlite3 = { - api: api, - SQLite3 - }; - - if(self === self.window){ - /* This is running in the main window thread, so we're done. */ - postMessage({type:'sqlite3-api',data:'loaded'}); - return; - } - /****************************************************************** - End of main window thread. What follows is only intended for use - in Worker threads. - ******************************************************************/ - - /** - UNDER CONSTRUCTION - - We need an API which can proxy the DB API via a Worker message - interface. The primary quirky factor in such an API is that we - cannot pass callback functions between the window thread and a - worker thread, so we have to receive all db results via - asynchronous message-passing. That requires an asychronous API - with a distinctly different shape that the main OO API. - - Certain important considerations here include: - - - Support only one db connection or multiple? The former is far - easier, but there's always going to be a user out there who - wants to juggle six database handles at once. Do we add that - complexity or tell such users to write their own code using - the provided lower-level APIs? - - - Fetching multiple results: do we pass them on as a series of - messages, with start/end messages on either end, or do we - collect all results and bundle them back in a single message? - The former is, generically speaking, more memory-efficient but - the latter far easier to implement in this environment. The - latter is untennable for large data sets. Despite a web page - hypothetically being a relatively limited environment, there - will always be those users who feel that they should/need to - be able to work with multi-hundred-meg (or larger) blobs, and - passing around arrays of those may quickly exhaust the JS - engine's memory. - - TODOs include, but are not limited to: - - - The ability to manage multiple DB handles. This can - potentially be done via a simple mapping of DB.filename or - DB._pDb (`sqlite3*` handle) to DB objects. The open() - interface would need to provide an ID (probably DB._pDb) back - to the user which can optionally be passed as an argument to - the other APIs (they'd default to the first-opened DB, for - ease of use). Client-side usability of this feature would - benefit from making another wrapper class (or a singleton) - available to the main thread, with that object proxying all(?) - communication with the worker. - - - Revisit how virtual files are managed. We currently delete DBs - from the virtual filesystem when we close them, for the sake - of saving memory (the VFS lives in RAM). Supporting multiple - DBs may require that we give up that habit. Similarly, fully - supporting ATTACH, where a user can upload multiple DBs and - ATTACH them, also requires the that we manage the VFS entries - better. As of this writing, ATTACH will fail fatally in the - fiddle app (but not the lower-level APIs) because it runs in - safe mode, where ATTACH is disabled. - */ - - /** - Helper for managing Worker-level state. - */ - const wState = { - db: undefined, - open: function(arg){ - if(!arg && this.db) return this.db; - else if(this.db) this.db.close(); - return this.db = (Array.isArray(arg) ? new DB(...arg) : new DB(arg)); - }, - close: function(alsoUnlink){ - if(this.db){ - this.db.close(alsoUnlink); - this.db = undefined; - } - }, - affirmOpen: function(){ - return this.db || toss("DB is not opened."); - }, - post: function(type,data,xferList){ - if(xferList){ - self.postMessage({type, data},xferList); - xferList.length = 0; - }else{ - self.postMessage({type, data}); - } - } - }; - - /** - A level of "organizational abstraction" for the Worker - API. Each method in this object must map directly to a Worker - message type key. The onmessage() dispatcher attempts to - dispatch all inbound messages to a method of this object, - passing it the event.data part of the inbound event object. All - methods must return a plain Object containing any response - state, which the dispatcher may amend. All methods must throw - on error. - */ - const wMsgHandler = { - xfer: [/*Temp holder for "transferable" postMessage() state.*/], - /** - Proxy for DB.exec() which expects a single argument of type - string (SQL to execute) or an options object in the form - expected by exec(). The notable differences from exec() - include: - - - The default value for options.rowMode is 'array' because - the normal default cannot cross the window/Worker boundary. - - - A function-type options.callback property cannot cross - the window/Worker boundary, so is not useful here. If - options.callback is a string then it is assumed to be a - message type key, in which case a callback function will be - applied which posts each row result via: - - postMessage({type: thatKeyType, data: theRow}) - - And, at the end of the result set (whether or not any - result rows were produced), it will post an identical - message with data:null to alert the caller than the result - set is completed. - - The callback proxy must not recurse into this interface, or - results are undefined. (It hypothetically cannot recurse - because an exec() call will be tying up the Worker thread, - causing any recursion attempt to wait until the first - exec() is completed.) - - The response is the input options object (or a synthesized - one if passed only a string), noting that - options.resultRows and options.columnNames may be populated - by the call to exec(). - - This opens/creates the Worker's db if needed. - */ - exec: function(ev){ - const opt = ( - 'string'===typeof ev.data - ) ? {sql: ev.data} : (ev.data || {}); - if(!opt.rowMode){ - /* Since the default rowMode of 'stmt' is not useful - for the Worker interface, we'll default to - something else. */ - opt.rowMode = 'array'; - }else if('stmt'===opt.rowMode){ - toss("Invalid rowMode for exec(): stmt mode", - "does not work in the Worker API."); - } - const db = wState.open(); - if(opt.callback || opt.resultRows instanceof Array){ - // Part of a copy-avoidance optimization for blobs - db._blobXfer = this.xfer; - } - const callbackMsgType = opt.callback; - if('string' === typeof callbackMsgType){ - const that = this; - opt.callback = - (row)=>wState.post(callbackMsgType,row,this.xfer); - } - try { - db.exec(opt); - if(opt.callback instanceof Function){ - opt.callback = callbackMsgType; - wState.post(callbackMsgType, null); - } - }finally{ - delete db._blobXfer; - if('string'===typeof callbackMsgType){ - opt.callback = callbackMsgType; - } - } - return opt; - }/*exec()*/, - /** - Proxy for DB.exportBinaryImage(). Throws if the db has not - been opened. Response is an object: - - { - buffer: Uint8Array (db file contents), - filename: the current db filename, - mimetype: string - } - */ - export: function(ev){ - const db = wState.affirmOpen(); - const response = { - buffer: db.exportBinaryImage(), - filename: db.filename, - mimetype: 'application/x-sqlite3' - }; - this.xfer.push(response.buffer.buffer); - return response; - }/*export()*/, - /** - Proxy for the DB constructor. Expects to be passed a single - object or a falsy value to use defaults. The object may - have a filename property to name the db file (see the DB - constructor for peculiarities and transformations) and/or a - buffer property (a Uint8Array holding a complete database - file's contents). The response is an object: - - { - filename: db filename (possibly differing from the input) - } - - If the Worker's db is currently opened, this call closes it - before proceeding. - */ - open: function(ev){ - wState.close(/*true???*/); - const args = [], data = (ev.data || {}); - if(data.filename) args.push(data.filename); - if(data.buffer){ - args.push(data.buffer); - this.xfer.push(data.buffer.buffer); - } - const db = wState.open(args); - return {filename: db.filename}; - }, - /** - Proxy for DB.close(). If ev.data may either be a boolean or - an object with an `unlink` property. If that value is - truthy then the db file (if the db is currently open) will - be unlinked from the virtual filesystem, else it will be - kept intact. The response object is: - - {filename: db filename _if_ the db is is opened when this - is called, else the undefined value - } - */ - close: function(ev){ - const response = { - filename: wState.db && wState.db.filename - }; - if(wState.db){ - wState.close(!!(ev.data && 'object'===typeof ev.data) - ? ev.data.unlink : ev.data); - } - return response; - } - }/*wMsgHandler*/; - - /** - UNDER CONSTRUCTION! - - A subset of the DB API is accessible via Worker messages in the form: - - { type: apiCommand, - data: apiArguments } - - As a rule, these commands respond with a postMessage() of their - own in the same form, but will, if needed, transform the `data` - member to an object and may add state to it. The responses - always have an object-format `data` part. If the inbound `data` - is an object which has a `messageId` property, that property is - always mirrored in the result object, for use in client-side - dispatching of these asynchronous results. Exceptions thrown - during processing result in an `error`-type event with a - payload in the form: - - { - message: error string, - errorClass: class name of the error type, - input: ev.data, - [messageId: if set in the inbound message] - } - - The individual APIs are documented in the wMsgHandler object. - */ - self.onmessage = function(ev){ - ev = ev.data; - let response, evType = ev.type; - try { - if(wMsgHandler.hasOwnProperty(evType) && - wMsgHandler[evType] instanceof Function){ - response = wMsgHandler[evType](ev); - }else{ - toss("Unknown db worker message type:",ev.type); - } - }catch(err){ - evType = 'error'; - response = { - message: err.message, - errorClass: err.name, - input: ev - }; - } - if(!response.messageId && ev.data - && 'object'===typeof ev.data && ev.data.messageId){ - response.messageId = ev.data.messageId; - } - wState.post(evType, response, wMsgHandler.xfer); - }; - - postMessage({type:'sqlite3-api',data:'loaded'}); -})/*postRun.push(...)*/; diff --git a/ext/fiddle/sqlite3-worker.js b/ext/fiddle/sqlite3-worker.js deleted file mode 100644 index ed369ad35b..0000000000 --- a/ext/fiddle/sqlite3-worker.js +++ /dev/null @@ -1,44 +0,0 @@ -/* - 2022-05-23 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This is a JS Worker file for the main sqlite3 api. It loads - sqlite3.js, initializes the module, and postMessage()'s a message - after the module is initialized: - - {type: 'sqlite3-api', data: 'ready'} - - This seemingly superfluous level of indirection is necessary when - loading sqlite3.js via a Worker. Loading sqlite3.js from the main - window thread elides the Worker-specific API. Instantiating a worker - with new Worker("sqlite.js") will not (cannot) call - sqlite3InitModule() to initialize the module due to a - timing/order-of-operations conflict (and that symbol is not exported - in a way that a Worker loading it that way can see it). Thus JS - code wanting to load the sqlite3 Worker-specific API needs to pass - _this_ file (or equivalent) to the Worker constructor and then - listen for an event in the form shown above in order to know when - the module has completed initialization. sqlite3.js will fire a - similar event, with data:'loaded' as the final step in its loading - process. Whether or not we _really_ need both 'loaded' and 'ready' - events is unclear, but they are currently separate events primarily - for the sake of clarity in the timing of when it's okay to use the - loaded module. At the time the 'loaded' event is fired, it's - possible (but unknown and unknowable) that the emscripten-generated - module-setup infrastructure still has work to do. Thus it is - hypothesized that client code is better off waiting for the 'ready' - even before using the API. -*/ -"use strict"; -importScripts('sqlite3.js'); -sqlite3InitModule().then(function(){ - setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'ready'}), 0); -}); diff --git a/ext/fiddle/testing1.js b/ext/fiddle/testing1.js deleted file mode 100644 index 552df5d6f7..0000000000 --- a/ext/fiddle/testing1.js +++ /dev/null @@ -1,239 +0,0 @@ -/* - 2022-05-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - A basic test script for sqlite3-api.js. This file must be run in - main JS thread and sqlite3.js must have been loaded before it. -*/ -(function(){ - const T = self.SqliteTestUtil; - const log = console.log.bind(console); - const debug = console.debug.bind(console); - - const assert = function(condition, text) { - if (!condition) { - throw new Error('Assertion failed' + (text ? ': ' + text : '')); - } - }; - - const test1 = function(db,sqlite3){ - const api = sqlite3.api; - log("Basic sanity tests..."); - T.assert(db._pDb). - assert(0===api.sqlite3_extended_result_codes(db._pDb,1)); - let st = db.prepare( - new TextEncoder('utf-8').encode("select 3 as a") - /* Testing handling of Uint8Array input */ - ); - //debug("statement =",st); - T.assert(st._pStmt) - .assert(!st._mayGet) - .assert('a' === st.getColumnName(0)) - .assert(st === db._statements[st._pStmt]) - .assert(1===st.columnCount) - .assert(0===st.parameterCount) - .mustThrow(()=>st.bind(1,null)) - .assert(true===st.step()) - .assert(3 === st.get(0)) - .mustThrow(()=>st.get(1)) - .mustThrow(()=>st.get(0,~api.SQLITE_INTEGER)) - .assert(3 === st.get(0,api.SQLITE_INTEGER)) - .assert(3 === st.getInt(0)) - .assert('3' === st.get(0,api.SQLITE_TEXT)) - .assert('3' === st.getString(0)) - .assert(3.0 === st.get(0,api.SQLITE_FLOAT)) - .assert(3.0 === st.getFloat(0)) - .assert(st.get(0,api.SQLITE_BLOB) instanceof Uint8Array) - .assert(1===st.get(0,api.SQLITE_BLOB).length) - .assert(st.getBlob(0) instanceof Uint8Array) - .assert(3 === st.get([])[0]) - .assert(3 === st.get({}).a) - .assert(3 === st.getJSON(0)) - .assert(st._mayGet) - .assert(false===st.step()) - .assert(!st._mayGet) - ; - let pId = st._pStmt; - st.finalize(); - T.assert(!st._pStmt) - .assert(!db._statements[pId]); - - let list = []; - db.exec({ - sql:['CREATE TABLE t(a,b);', - "INSERT INTO t(a,b) VALUES(1,2),(3,4),", - "(?,?),('blob',X'6869');" - ], - multi: true, - saveSql: list, - bind: [5,6] - /* Achtung: ^^^ bind support might be removed from multi-mode exec. */ - }); - T.assert(2 === list.length); - //debug("Exec'd SQL:", list); - - let blob = db.selectValue("select b from t where a='blob'"); - T.assert(blob instanceof Uint8Array). - assert(0x68===blob[0] && 0x69===blob[1]); - blob = null; - - let counter = 0, colNames = []; - list.length = 0; - db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ - rowMode: 'object', - resultRows: list, - columnNames: colNames, - callback: function(row,stmt){ - ++counter; - T.assert((row.a%2 && row.a<6) || 'blob'===row.a); - } - }); - T.assert(2 === colNames.length) - .assert('a' === colNames[0]) - .assert(4 === counter) - .assert(4 === list.length); - list.length = 0; - db.exec("SELECT a a, b b FROM t",{ - rowMode: 'array', - callback: function(row,stmt){ - ++counter; - T.assert(Array.isArray(row)) - .assert((0===row[1]%2 && row[1]<7) - || (row[1] instanceof Uint8Array)); - } - }); - T.assert(8 === counter); - }; - - const testUDF = function(db){ - log("Testing UDF..."); - db.createFunction("foo",function(a,b){return a+b}); - T.assert(7===db.selectValue("select foo(3,4)")). - assert(5===db.selectValue("select foo(3,?)",2)). - assert(5===db.selectValue("select foo(?,?2)",[1,4])). - assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5})); - db.createFunction("bar", { - arity: -1, - callback: function(){ - var rc = 0; - for(let i = 0; i < arguments.length; ++i) rc += arguments[i]; - return rc; - } - }).createFunction({ - name: "asis", - callback: (arg)=>arg - }); - - log("Testing DB::selectValue() w/ UDF..."); - T.assert(0===db.selectValue("select bar()")). - assert(1===db.selectValue("select bar(1)")). - assert(3===db.selectValue("select bar(1,2)")). - assert(-1===db.selectValue("select bar(1,2,-4)")). - assert('hi'===db.selectValue("select asis('hi')")); - - const eqApprox = function(v1,v2,factor=0.05){ - //debug('eqApprox',v1, v2); - return v1>=(v2-factor) && v1<=(v2+factor); - }; - - T.assert('hi' === db.selectValue("select ?",'hi')). - assert(null===db.selectValue("select null")). - assert(null === db.selectValue("select ?",null)). - assert(null === db.selectValue("select ?",[null])). - assert(null === db.selectValue("select $a",{$a:null})). - assert(eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). - assert(eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))) - ; - - log("Testing binding and UDF propagation of blobs..."); - let blobArg = new Uint8Array(2); - blobArg.set([0x68, 0x69], 0); - let blobRc = db.selectValue("select asis(?1)", blobArg); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length). - assert(0x68==blobRc[0] && 0x69==blobRc[1]); - blobRc = db.selectValue("select asis(X'6869')"); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length). - assert(0x68==blobRc[0] && 0x69==blobRc[1]); - - blobArg = new Int8Array(2); - blobArg.set([0x68, 0x69]); - //debug("blobArg=",blobArg); - blobRc = db.selectValue("select asis(?1)", blobArg); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length); - //debug("blobRc=",blobRc); - T.assert(0x68==blobRc[0] && 0x69==blobRc[1]); - }; - - const testAttach = function(db){ - log("Testing ATTACH..."); - db.exec({ - sql:[ - "attach 'foo.db' as foo", - "create table foo.bar(a)", - "insert into foo.bar(a) values(1),(2),(3)" - ].join(';'), - multi: true - }); - T.assert(2===db.selectValue('select a from foo.bar where a>1 order by a')); - db.exec("detach foo"); - T.mustThrow(()=>db.exec("select * from foo.bar")); - }; - - const runTests = function(Module){ - const sqlite3 = Module.sqlite3; - const api = sqlite3.api; - const oo = sqlite3.SQLite3; - log("Loaded module:",api.sqlite3_libversion(), - api.sqlite3_sourceid()); - log("Build options:",oo.compileOptionUsed()); - log("api.wasm.HEAP8 size =",api.wasm.HEAP8().length); - log("wasmEnum",JSON.parse(Module.ccall('sqlite3_wasm_enum_json', 'string', []))); - [ /* Spot-check a handful of constants to make sure they got installed... */ - 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', - 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', - 'SQLITE_OPEN_CREATE' - ].forEach(function(k){ - T.assert('number' === typeof api[k]); - }); - [/* Spot-check a few of the WASM API methods. */ - '_free', '_malloc', 'addFunction', 'stackRestore' - ].forEach(function(k){ - T.assert(Module[k] instanceof Function). - assert(api.wasm[k] instanceof Function); - }); - - const db = new oo.DB(); - try { - log("DB:",db.filename); - [ - test1, testUDF, testAttach - ].forEach((f)=>{ - const t = T.counter; - f(db, sqlite3); - log("Test count:",T.counter - t); - }); - }finally{ - db.close(); - } - log("Total Test count:",T.counter); - }; - - sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){ - /** Use a timeout so that we are (hopefully) out from under - the module init stack when our setup gets run. Just on - principle, not because we _need_ to be. */ - //console.debug("theModule =",theModule); - setTimeout(()=>runTests(theModule), 0); - }); -})(); diff --git a/ext/fiddle/testing2.js b/ext/fiddle/testing2.js deleted file mode 100644 index 4bb21b95f0..0000000000 --- a/ext/fiddle/testing2.js +++ /dev/null @@ -1,235 +0,0 @@ -/* - 2022-05-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - A basic test script for sqlite3-worker.js. -*/ -(function(){ - const T = self.SqliteTestUtil; - const SW = new Worker("sqlite3-worker.js"); - /** Posts a worker message as {type:type, data:data}. */ - const wMsg = function(type,data){ - SW.postMessage({type, data}); - return SW; - }; - const log = console.log.bind(console); - const warn = console.warn.bind(console); - const error = console.error.bind(console); - - SW.onerror = function(event){ - error("onerror",event); - }; - - /** - A queue for callbacks which are to be run in response to async - DB commands. See the notes in runTests() for why we need - this. The event-handling plumbing of this file requires that - any DB command which includes a `messageId` property also have - a queued callback entry, as the existence of that property in - response payloads is how it knows whether or not to shift an - entry off of the queue. - */ - const MsgHandlerQueue = { - queue: [], - id: 0, - push: function(type,callback){ - this.queue.push(callback); - return type + '-' + (++this.id); - }, - shift: function(){ - return this.queue.shift(); - } - }; - - const testCount = ()=>log("Total test count:",T.counter); - - const runOneTest = function(eventType, eventData, callback){ - T.assert(eventData && 'object'===typeof eventData); - /* ^^^ that is for the testing and messageId-related code, not - a hard requirement of all of the Worker-exposed APIs. */ - eventData.messageId = MsgHandlerQueue.push(eventType,function(ev){ - log("runOneTest",eventType,"result",ev.data); - if(callback instanceof Function){ - callback(ev); - testCount(); - } - }); - wMsg(eventType, eventData); - }; - - /** Methods which map directly to onmessage() event.type keys. - They get passed the inbound event.data. */ - const dbMsgHandler = { - open: function(ev){ - log("open result",ev.data); - }, - exec: function(ev){ - log("exec result",ev.data); - }, - export: function(ev){ - log("exec result",ev.data); - }, - error: function(ev){ - error("ERROR from the worker:",ev.data); - }, - resultRowTest1: function f(ev){ - if(undefined === f.counter) f.counter = 0; - if(ev.data) ++f.counter; - //log("exec() result row:",ev.data); - T.assert(null===ev.data || 'number' === typeof ev.data.b); - } - }; - - const runTests = function(){ - const mustNotReach = ()=>{ - throw new Error("This is not supposed to be reached."); - }; - /** - "The problem" now is that the test results are async. We - know, however, that the messages posted to the worker will - be processed in the order they are passed to it, so we can - create a queue of callbacks to handle them. The problem - with that approach is that it's not error-handling - friendly, in that an error can cause us to bypass a result - handler queue entry. We have to perform some extra - acrobatics to account for that. - */ - runOneTest('open', {filename:'testing2.sqlite3'}, function(ev){ - //log("open result",ev); - T.assert('testing2.sqlite3'===ev.data.filename) - .assert(ev.data.messageId); - }); - runOneTest('exec',{ - sql: ["create table t(a,b)", - "insert into t(a,b) values(1,2),(3,4),(5,6)" - ].join(';'), - multi: true, - resultRows: [], columnNames: [] - }, function(ev){ - ev = ev.data; - T.assert(0===ev.resultRows.length) - .assert(0===ev.columnNames.length); - }); - runOneTest('exec',{ - sql: 'select a a, b b from t order by a', - resultRows: [], columnNames: [], - }, function(ev){ - ev = ev.data; - T.assert(3===ev.resultRows.length) - .assert(1===ev.resultRows[0][0]) - .assert(6===ev.resultRows[2][1]) - .assert(2===ev.columnNames.length) - .assert('b'===ev.columnNames[1]); - }); - runOneTest('exec',{ - sql: 'select a a, b b from t order by a', - resultRows: [], columnNames: [], - rowMode: 'object' - }, function(ev){ - ev = ev.data; - T.assert(3===ev.resultRows.length) - .assert(1===ev.resultRows[0].a) - .assert(6===ev.resultRows[2].b) - }); - runOneTest('exec',{sql:'intentional_error'}, mustNotReach); - // Ensure that the message-handler queue survives ^^^ that error... - runOneTest('exec',{ - sql:'select 1', - resultRows: [], - //rowMode: 'array', // array is the default in the Worker interface - }, function(ev){ - ev = ev.data; - T.assert(1 === ev.resultRows.length) - .assert(1 === ev.resultRows[0][0]); - }); - runOneTest('exec',{ - sql: 'select a a, b b from t order by a', - callback: 'resultRowTest1', - rowMode: 'object' - }, function(ev){ - T.assert(3===dbMsgHandler.resultRowTest1.counter); - dbMsgHandler.resultRowTest1.counter = 0; - }); - runOneTest('exec',{sql: 'delete from t where a>3'}); - runOneTest('exec',{ - sql: 'select count(a) from t', - resultRows: [] - },function(ev){ - ev = ev.data; - T.assert(1===ev.resultRows.length) - .assert(2===ev.resultRows[0][0]); - }); - runOneTest('export',{}, function(ev){ - ev = ev.data; - T.assert('string' === typeof ev.filename) - .assert(ev.buffer instanceof Uint8Array) - .assert(ev.buffer.length > 1024) - .assert('application/x-sqlite3' === ev.mimetype); - }); - - /***** close() tests must come last. *****/ - runOneTest('close',{unlink:true},function(ev){ - ev = ev.data; - T.assert('string' === typeof ev.filename); - }); - runOneTest('close',{unlink:true},function(ev){ - ev = ev.data; - T.assert(undefined === ev.filename); - }); - }; - - SW.onmessage = function(ev){ - if(!ev.data || 'object'!==typeof ev.data){ - warn("Unknown sqlite3-worker message type:",ev); - return; - } - ev = ev.data/*expecting a nested object*/; - //log("main window onmessage:",ev); - if(ev.data && ev.data.messageId){ - /* We're expecting a queued-up callback handler. */ - const f = MsgHandlerQueue.shift(); - if('error'===ev.type){ - dbMsgHandler.error(ev); - return; - } - T.assert(f instanceof Function); - f(ev); - return; - } - switch(ev.type){ - case 'sqlite3-api': - switch(ev.data){ - case 'loaded': - log("Message:",ev); return; - case 'ready': - log("Message:",ev); - self.sqlite3TestModule.setStatus(null); - setTimeout(runTests, 0); - return; - default: - warn("Unknown sqlite3-api message type:",ev); - return; - } - default: - if(dbMsgHandler.hasOwnProperty(ev.type)){ - try{dbMsgHandler[ev.type](ev);} - catch(err){ - error("Exception while handling db result message", - ev,":",err); - } - return; - } - warn("Unknown sqlite3-api message type:",ev); - } - }; - - log("Init complete, but async init bits may still be running."); -})(); diff --git a/ext/fiddle/wasm_util.c b/ext/fiddle/wasm_util.c deleted file mode 100644 index a02063fc48..0000000000 --- a/ext/fiddle/wasm_util.c +++ /dev/null @@ -1,127 +0,0 @@ -#include "sqlite3.h" -#include /*atexit()*/ -/* -** 2022-06-25 -** -** The author disclaims copyright to this source code. In place of a -** legal notice, here is a blessing: -** -** * May you do good and not evil. -** * May you find forgiveness for yourself and forgive others. -** * May you share freely, never taking more than you give. -** -*********************************************************************** -** -** Utility functions for use with the emscripten/WASM bits. These -** functions ARE NOT part of the sqlite3 public API. They are strictly -** for internal use by the JS/WASM bindings. -** -** This file is intended to be WASM-compiled together with sqlite3.c, -** e.g.: -** -** emcc ... sqlite3.c wasm_util.c -*/ - -/** Result value of sqlite3_wasm_enum_json(). */ -static char * zWasmEnum = 0; -/* atexit() handler to clean up any WASM-related state. */ -static void sqlite3_wasm_cleanup(void){ - free(zWasmEnum); -} - -/* -** Returns a string containing a JSON-format "enum" of C-level -** constants intended to be imported into the JS environment. The JSON -** is initialized the first time this function is called and that -** result is reused for all future calls and cleaned up via atexit(). -** (If we didn't cache the result, it would be leaked by the JS glue -** code on each call during the WASM-to-JS conversion.) -** -** This function is NOT part of the sqlite3 public API. It is strictly -** for use by the JS/WASM bindings. -*/ -const char * sqlite3_wasm_enum_json(void){ - sqlite3_str * s; - if(zWasmEnum) return zWasmEnum; - s = sqlite3_str_new(0); - sqlite3_str_appendall(s, "{"); - -#define SD_(X,S,FINAL) \ - sqlite3_str_appendf(s, "\"%s\": %d%s", S, (int)X, (FINAL ? "}" : ", ")) -#define SD(X) SD_(X,#X,0) -#define SDFinal(X) SD_(X,#X,1) - - sqlite3_str_appendall(s,"\"resultCodes\": {"); - SD(SQLITE_OK); - SD(SQLITE_ERROR); - SD(SQLITE_INTERNAL); - SD(SQLITE_PERM); - SD(SQLITE_ABORT); - SD(SQLITE_BUSY); - SD(SQLITE_LOCKED); - SD(SQLITE_NOMEM); - SD(SQLITE_READONLY); - SD(SQLITE_INTERRUPT); - SD(SQLITE_IOERR); - SD(SQLITE_CORRUPT); - SD(SQLITE_NOTFOUND); - SD(SQLITE_FULL); - SD(SQLITE_CANTOPEN); - SD(SQLITE_PROTOCOL); - SD(SQLITE_EMPTY); - SD(SQLITE_SCHEMA); - SD(SQLITE_TOOBIG); - SD(SQLITE_CONSTRAINT); - SD(SQLITE_MISMATCH); - SD(SQLITE_MISUSE); - SD(SQLITE_NOLFS); - SD(SQLITE_AUTH); - SD(SQLITE_FORMAT); - SD(SQLITE_RANGE); - SD(SQLITE_NOTADB); - SD(SQLITE_NOTICE); - SD(SQLITE_WARNING); - SD(SQLITE_ROW); - SDFinal(SQLITE_DONE); - - sqlite3_str_appendall(s,",\"dataTypes\": {"); - SD(SQLITE_INTEGER); - SD(SQLITE_FLOAT); - SD(SQLITE_TEXT); - SD(SQLITE_BLOB); - SDFinal(SQLITE_NULL); - - sqlite3_str_appendf(s,",\"encodings\": {"); - SDFinal(SQLITE_UTF8); - - sqlite3_str_appendall(s,",\"blobFinalizers\": {"); - SD(SQLITE_STATIC); - SDFinal(SQLITE_TRANSIENT); - - sqlite3_str_appendall(s,",\"udfFlags\": {"); - SD(SQLITE_DETERMINISTIC); - SD(SQLITE_DIRECTONLY); - SDFinal(SQLITE_INNOCUOUS); - - sqlite3_str_appendall(s,",\"openFlags\": {"); - /* Noting that not all of these will have any effect in WASM-space. */ - SD(SQLITE_OPEN_READONLY); - SD(SQLITE_OPEN_READWRITE); - SD(SQLITE_OPEN_CREATE); - SD(SQLITE_OPEN_URI); - SD(SQLITE_OPEN_MEMORY); - SD(SQLITE_OPEN_NOMUTEX); - SD(SQLITE_OPEN_FULLMUTEX); - SD(SQLITE_OPEN_SHAREDCACHE); - SD(SQLITE_OPEN_PRIVATECACHE); - SD(SQLITE_OPEN_EXRESCODE); - SDFinal(SQLITE_OPEN_NOFOLLOW); - -#undef SD_ -#undef SD -#undef SDFinal - sqlite3_str_appendall(s, "}"); - zWasmEnum = sqlite3_str_finish(s); - atexit(sqlite3_wasm_cleanup); - return zWasmEnum; -} diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 99a1c5b031..ee8ade74a3 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -1,11 +1,269 @@ # This GNU makefile exists primarily to simplify/speed up development -# from emacs. It is not part of the canonical build process. -default: - $(MAKE) -C ../.. wasm -e emcc_opt=-O0 +# of the sqlite3 WASM components. It is not part of the canonical +# build process. +# +# Maintenance notes: the fiddle build is currently performed in the +# top-level ../../Makefile.in. It may be moved into this file at some +# point, as GNU Make has been deemed acceptable for the WASM-related +# components (whereas POSIX Make is required for the more conventional +# components). +SHELL := $(shell which bash 2>/dev/null) +all: + +.PHONY: fiddle +ifneq (,$(wildcard /home/stephan)) + fiddle_opt ?= -O0 +else + fiddle_opt = -Os +endif +fiddle: + $(MAKE) -C ../.. fiddle -e emcc_opt=$(fiddle_opt) clean: - $(MAKE) -C ../../ clean-wasm + $(MAKE) -C ../../ clean-fiddle + -rm -f $(CLEAN_FILES) +MAKEFILE := $(lastword $(MAKEFILE_LIST)) +dir.top := ../.. +# Reminder: some Emscripten flags require absolute paths +dir.wasm := $(patsubst %/,%,$(dir $(abspath $(MAKEFILE)))) +dir.api := api +dir.jacc := jaccwabyt +dir.common := common +CLEAN_FILES := *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ + +SQLITE_OPT = \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_UTF16 \ + -DSQLITE_THREADSAFE=0 +#SQLITE_OPT += -DSQLITE_ENABLE_MEMSYS5 +$(dir.top)/sqlite3.c: + $(MAKE) -C $(dir.top) sqlite3.c + +# SQLITE_OMIT_LOAD_EXTENSION: if this is true, sqlite3_vfs::xDlOpen +# and friends may be NULL. + +emcc_opt ?= -O0 +.PHONY: release +release: + $(MAKE) 'emcc_opt=-Os -g3' +# ^^^^^ target-specific vars, e.g.: +# release: emcc_opt=... +# apparently only work for file targets, not PHONY targets? +# +# ^^^^ -O3, -Oz, -Os minify symbol names and there appears to be no +# way around that except to use -g3, but -g3 causes the binary file +# size to absolutely explode (approx. 5x larger). This minification +# utterly breaks the resulting module, making it unsable except as +# self-contained/self-referential-only code, as ALL of the exported +# symbols get minified names. +# +# However, we have an option for using -Oz or -Os: +# +# Build with (-Os -g3) or (-Oz -g3) then use wasm-strip, from the wabt +# tools package (https://github.com/WebAssembly/wabt), to strip the +# debugging symbols. That results in a small build with unmangled +# symbol names. -Oz gives ever-so-slightly better compression than +# -Os: not quite 1% in some completely unscientific tests. Runtime +# speed for the unit tests is all over the place either way so it's +# difficult to say whether -Os gives any speed benefit over -Oz. +######################################################################## + +# Emscripten SDK home dir and related binaries... +EMSDK_HOME ?= $(word 1,$(wildcard $(HOME)/src/emsdk $(HOME)/emsdk)) +emcc.bin ?= $(word 1,$(wildcard $(shell which emcc) $(EMSDK_HOME)/upstream/emscripten/emcc)) +ifeq (,$(emcc.bin)) + $(error Cannot find emcc.) +endif + +wasm-strip ?= $(shell which wasm-strip 2>/dev/null) +ifeq (,$(filter clean,$(MAKECMDGOALS))) +ifeq (,$(wasm-strip)) + $(info WARNING: *******************************************************************) + $(info WARNING: builds using -O3/-Os/-Oz will minify WASM-exported names,) + $(info WARNING: breaking _All The Things_. The workaround for that is to build) + $(info WARNING: with -g3 (which explodes the file size) and then strip the debug) + $(info WARNING: info after compilation, using wasm-strip, to shrink the wasm file.) + $(info WARNING: wasm-strip was not found in the PATH so we cannot strip those.) + $(info WARNING: If this build uses any optimization level higher than -O2 then) + $(info WARNING: the ***resulting WASM binary WILL NOT BE USABLE***.) + $(info WARNING: wasm-strip is part of the wabt package:) + $(info WARNING: https://github.com/WebAssembly/wabt) + $(info WARNING: on Ubuntu-like systems it can be installed with:) + $(info WARNING: sudo apt install wabt) + $(info WARNING: *******************************************************************) +endif +endif # 'make clean' check + +ifeq (release,$(filter release,$(MAKECMDGOALS))) + ifeq (,$(wasm-strip)) + $(error Cannot make release-quality binary because wasm-strip is not available. \ + See notes in the warning above) + endif +else + $(info Development build. Use '$(MAKE) release' for a smaller release build.) +endif + +EXPORTED_FUNCTIONS.api.in := $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api \ + $(dir.jacc)/jaccwabyt_test.exports + +EXPORTED_FUNCTIONS.api: $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) + cat $(EXPORTED_FUNCTIONS.api.in) > $@ +CLEAN_FILES += EXPORTED_FUNCTIONS.api + +sqlite3-api.jses := \ + $(dir.api)/sqlite3-api-prologue.js \ + $(dir.common)/whwasmutil.js \ + $(dir.jacc)/jaccwabyt.js \ + $(dir.api)/sqlite3-api-glue.js \ + $(dir.api)/sqlite3-api-oo1.js \ + $(dir.api)/sqlite3-api-worker.js \ + $(dir.api)/sqlite3-api-opfs.js \ + $(dir.api)/sqlite3-api-cleanup.js + +sqlite3-api.js := $(dir.api)/sqlite3-api.js +CLEAN_FILES += $(sqlite3-api.js) +$(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE) + @echo "Making $@..." + @for i in $(sqlite3-api.jses); do \ + echo "/* BEGIN FILE: $$i */"; \ + cat $$i; \ + echo "/* END FILE: $$i */"; \ + done > $@ + +post-js.js := $(dir.api)/post-js.js +CLEAN_FILES += $(post-js.js) +post-jses := \ + $(dir.api)/post-js-header.js \ + $(sqlite3-api.js) \ + $(dir.api)/post-js-footer.js + +$(post-js.js): $(post-jses) $(MAKEFILE) + @echo "Making $@..." + @for i in $(post-jses); do \ + echo "/* BEGIN FILE: $$i */"; \ + cat $$i; \ + echo "/* END FILE: $$i */"; \ + done > $@ + + +######################################################################## +# emcc flags for .c/.o/.wasm. +emcc.flags = +#emcc.flags += -v # _very_ loud but also informative about what it's doing + +######################################################################## +# emcc flags for .c/.o. +emcc.cflags := +emcc.cflags += -std=c99 -fPIC +# -------------^^^^^^^^ we currently need c99 for WASM-specific sqlite3 APIs. +emcc.cflags += -I. -I$(dir.top) # $(SQLITE_OPT) + +######################################################################## +# emcc flags specific to building the final .js/.wasm file... +emcc.jsflags := -fPIC +emcc.jsflags += --no-entry +emcc.jsflags += -sENVIRONMENT=web +emcc.jsflags += -sMODULARIZE +emcc.jsflags += -sSTRICT_JS +emcc.jsflags += -sDYNAMIC_EXECUTION=0 +emcc.jsflags += -sNO_POLYFILL +emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api +emcc.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory # wasmMemory==>for -sIMPORTED_MEMORY +emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 +emcc.jsflags += -sIMPORTED_MEMORY +#emcc.jsflags += -sINITIAL_MEMORY=13107200 +#emcc.jsflags += -sTOTAL_STACK=4194304 +emcc.jsflags += -sEXPORT_NAME=sqlite3InitModule +emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. +emcc.jsflags +=--post-js=$(post-js.js) +#emcc.jsflags += -sSTRICT # fails due to missing __syscall_...() +#emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS +#emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API +#emcc.jsflags += -sABORTING_MALLOC +emcc.jsflags += -sALLOW_MEMORY_GROWTH +emcc.jsflags += -sALLOW_TABLE_GROWTH +emcc.jsflags += -Wno-limited-postlink-optimizations +# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. +#emcc.jsflags += -sMALLOC=emmalloc +#emcc.jsflags += -sMALLOC=dlmalloc # a good 8k larger than emmalloc +#emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why +#emcc.jsflags += --import=foo_bar +#emcc.jsflags += --no-gc-sections +# https://lld.llvm.org/WebAssembly.html +emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0 +emcc.jsflags += -sLLD_REPORT_UNDEFINED +#emcc.jsflags += --allow-undefined +emcc.jsflags += --import-undefined +#emcc.jsflags += --unresolved-symbols=import-dynamic --experimental-pic +#emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined +#emcc.jsflags += --unresolved-symbols=ignore-all +enable_bigint ?= 1 +ifneq (0,$(enable_bigint)) +emcc.jsflags += -sWASM_BIGINT +endif +emcc.jsflags += -sMEMORY64=0 +# ^^^^ MEMORY64=1 fails to load, erroring with: +# invalid memory limits flags 0x5 +# (enable via --experimental-wasm-memory64) +# +# ^^^^ MEMORY64=2 builds and loads but dies when we do things like: +# +# new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n) +# +# because ptr is now a BigInt, so is invalid for passing to arguments +# which have strict must-be-a-number requirements. +######################################################################## + + +sqlite3.js := $(dir.api)/sqlite3.js +sqlite3.wasm := $(dir.api)/sqlite3.wasm +$(dir.api)/sqlite3-wasm.o: emcc.cflags += $(SQLITE_OPT) +$(dir.api)/sqlite3-wasm.o: $(dir.top)/sqlite3.c +$(dir.api)/wasm_util.o: emcc.cflags += $(SQLITE_OPT) +sqlite3.wasm.c := $(dir.api)/sqlite3-wasm.c \ + $(dir.jacc)/jaccwabyt_test.c +# ^^^ FIXME (how?): jaccwabyt_test.c is only needed for the test +# apps. However, we want to test the release builds with those apps, +# so we cannot simply elide that file in release builds. That +# component is critical to the VFS bindings so needs to be tested +# along with the core APIs. +define WASM_C_COMPILE +$(1).o := $$(subst .c,.o,$(1)) +sqlite3.wasm.obj += $$($(1).o) +$$($(1).o): $$(MAKEFILE) $(1) + $$(emcc.bin) $$(emcc_opt) $$(emcc.flags) $$(emcc.cflags) -c $(1) -o $$@ +CLEAN_FILES += $$($(1).o) +endef +$(foreach c,$(sqlite3.wasm.c),$(eval $(call WASM_C_COMPILE,$(c)))) +$(sqlite3.js): +$(sqlite3.js): $(MAKEFILE) $(sqlite3.wasm.obj) \ + EXPORTED_FUNCTIONS.api \ + $(post-js.js) + $(emcc.bin) -o $@ $(emcc_opt) $(emcc.flags) $(emcc.jsflags) $(sqlite3.wasm.obj) + chmod -x $(sqlite3.wasm) +ifneq (,$(wasm-strip)) + $(wasm-strip) $(sqlite3.wasm) +endif + @ls -la $@ $(sqlite3.wasm) + +CLEAN_FILES += $(sqlite3.js) $(sqlite3.wasm) +all: $(sqlite3.js) +# End main Emscripten-based module build +######################################################################## + + +######################################################################## # fiddle_remote is the remote destination for the fiddle app. It # must be a [user@]HOST:/path for rsync. # Note that the target "should probably" contain a symlink of @@ -18,12 +276,12 @@ else ifneq (,$(wildcard /home/drh)) #fiddle_remote = if appropriate, add that user@host:/path here endif endif - $(fiddle_files): default - push-fiddle: $(fiddle_files) @if [ x = "x$(fiddle_remote)" ]; then \ echo "fiddle_remote must be a [user@]HOST:/path for rsync"; \ exit 1; \ fi rsync -va fiddle/ $(fiddle_remote) +# end fiddle remote push +######################################################################## diff --git a/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api similarity index 71% rename from ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api rename to ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index dae46f0359..8f103c7c0b 100644 --- a/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -7,6 +7,7 @@ _sqlite3_bind_parameter_count _sqlite3_bind_parameter_index _sqlite3_bind_text _sqlite3_changes +_sqlite3_changes64 _sqlite3_clear_bindings _sqlite3_close_v2 _sqlite3_column_blob @@ -24,30 +25,48 @@ _sqlite3_compileoption_used _sqlite3_create_function_v2 _sqlite3_data_count _sqlite3_db_filename +_sqlite3_db_name _sqlite3_errmsg +_sqlite3_error_offset +_sqlite3_errstr _sqlite3_exec +_sqlite3_expanded_sql +_sqlite3_extended_errcode _sqlite3_extended_result_codes _sqlite3_finalize +_sqlite3_initialize _sqlite3_interrupt _sqlite3_libversion +_sqlite3_libversion_number _sqlite3_open _sqlite3_open_v2 _sqlite3_prepare_v2 -_sqlite3_prepare_v2 +_sqlite3_prepare_v3 _sqlite3_reset _sqlite3_result_blob _sqlite3_result_double _sqlite3_result_error +_sqlite3_result_error_code +_sqlite3_result_error_nomem +_sqlite3_result_error_toobig _sqlite3_result_int _sqlite3_result_null _sqlite3_result_text _sqlite3_sourceid _sqlite3_sql _sqlite3_step +_sqlite3_strglob +_sqlite3_strlike +_sqlite3_total_changes +_sqlite3_total_changes64 _sqlite3_value_blob _sqlite3_value_bytes _sqlite3_value_double _sqlite3_value_text _sqlite3_value_type +_sqlite3_vfs_find +_sqlite3_vfs_register +_sqlite3_wasm_db_error _sqlite3_wasm_enum_json +_malloc _free diff --git a/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api b/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api new file mode 100644 index 0000000000..aab1d8bd37 --- /dev/null +++ b/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api @@ -0,0 +1,3 @@ +FS +wasmMemory + diff --git a/ext/wasm/api/post-js-footer.js b/ext/wasm/api/post-js-footer.js new file mode 100644 index 0000000000..ee470928ba --- /dev/null +++ b/ext/wasm/api/post-js-footer.js @@ -0,0 +1,3 @@ +/* The current function scope was opened via post-js-header.js, which + gets prepended to this at build-time. */ +})/*postRun.push(...)*/; diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js new file mode 100644 index 0000000000..1763188a21 --- /dev/null +++ b/ext/wasm/api/post-js-header.js @@ -0,0 +1,26 @@ +/** + post-js-header.js is to be prepended to other code to create + post-js.js for use with Emscripten's --post-js flag. This code + requires that it be running in that context. The Emscripten + environment must have been set up already but it will not have + loaded its WASM when the code in this file is run. The function it + installs will be run after the WASM module is loaded, at which + point the sqlite3 WASM API bits will be set up. +*/ +if(!Module.postRun) Module.postRun = []; +Module.postRun.push(function(Module/*the Emscripten-style module object*/){ + 'use strict'; + /* This function will contain: + + - post-js-header.js (this file) + - sqlite3-api-prologue.js => Bootstrapping bits to attach the rest to + - sqlite3-api-whwasmutil.js => Replacements for much of Emscripten's glue + - sqlite3-api-jaccwabyt.js => Jaccwabyt (C/JS struct binding) + - sqlite3-api-glue.js => glues previous parts together + - sqlite3-api-oo.js => SQLite3 OO API #1. + - sqlite3-api-worker.js => Worker-based API + - sqlite3-api-cleanup.js => final API cleanup + - post-js-footer.js => closes this postRun() function + + Whew! + */ diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js new file mode 100644 index 0000000000..a2f921a5d7 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -0,0 +1,44 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is the tail end of the sqlite3-api.js constellation, + intended to be appended after all other files so that it can clean + up any global systems temporarily used for setting up the API's + various subsystems. +*/ +'use strict'; +self.sqlite3.postInit.forEach( + self.importScripts/*global is a Worker*/ + ? function(f){ + /** We try/catch/report for the sake of failures which happen in + a Worker, as those exceptions can otherwise get completely + swallowed, leading to confusing downstream errors which have + nothing to do with this failure. */ + try{ f(self, self.sqlite3) } + catch(e){ + console.error("Error in postInit() function:",e); + throw e; + } + } + : (f)=>f(self, self.sqlite3) +); +delete self.sqlite3.postInit; +if(self.location && +self.location.port > 1024){ + console.warn("Installing sqlite3 bits as global S for dev-testing purposes."); + self.S = self.sqlite3; +} +/* Clean up temporary global-scope references to our APIs... */ +self.sqlite3.config.Module.sqlite3 = self.sqlite3 +/* ^^^^ Currently needed by test code and Worker API setup */; +delete self.sqlite3.capi.util /* arguable, but these are (currently) internal-use APIs */; +delete self.sqlite3 /* clean up our global-scope reference */; +//console.warn("Module.sqlite3 =",Module.sqlite3); diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js new file mode 100644 index 0000000000..e962c93b64 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -0,0 +1,211 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file glues together disparate pieces of JS which are loaded in + previous steps of the sqlite3-api.js bootstrapping process: + sqlite3-api-prologue.js, whwasmutil.js, and jaccwabyt.js. It + initializes the main API pieces so that the downstream components + (e.g. sqlite3-api-oo1.js) have all that they need. +*/ +(function(self){ + 'use strict'; + const toss = (...args)=>{throw new Error(args.join(' '))}; + + self.sqlite3 = self.sqlite3ApiBootstrap({ + Module: Module /* ==> Emscripten-style Module object. Currently + needs to be exposed here for test code. NOT part + of the public API. */, + exports: Module['asm'], + memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */, + bigIntEnabled: !!self.BigInt64Array, + allocExportName: 'malloc', + deallocExportName: 'free' + }); + delete self.sqlite3ApiBootstrap; + + const sqlite3 = self.sqlite3; + const capi = sqlite3.capi, wasm = capi.wasm, util = capi.util; + self.WhWasmUtilInstaller(capi.wasm); + delete self.WhWasmUtilInstaller; + + if(0){ + /* "The problem" is that the following isn't type-safe. + OTOH, nothing about WASM pointers is. */ + /** + Add the `.pointer` xWrap() signature entry to extend + the `pointer` arg handler to check for a `pointer` + property. This can be used to permit, e.g., passing + an SQLite3.DB instance to a C-style sqlite3_xxx function + which takes an `sqlite3*` argument. + */ + const oldP = wasm.xWrap.argAdapter('pointer'); + const adapter = function(v){ + if(v && 'object'===typeof v && v.constructor){ + const x = v.pointer; + if(Number.isInteger(x)) return x; + else toss("Invalid (object) type for pointer-type argument."); + } + return oldP(v); + }; + wasm.xWrap.argAdapter('.pointer', adapter); + } + + // WhWasmUtil.xWrap() bindings... + { + /** + Add some descriptive xWrap() aliases for '*' intended to + (A) initially improve readability/correctness of capi.signatures + and (B) eventually perhaps provide some sort of type-safety + in their conversions. + */ + const aPtr = wasm.xWrap.argAdapter('*'); + wasm.xWrap.argAdapter('sqlite3*', aPtr)('sqlite3_stmt*', aPtr); + + /** + Populate api object with sqlite3_...() by binding the "raw" wasm + exports into type-converting proxies using wasm.xWrap(). + */ + for(const e of wasm.bindingSignatures){ + capi[e[0]] = wasm.xWrap.apply(null, e); + } + + /* For functions which cannot work properly unless + wasm.bigIntEnabled is true, install a bogus impl which + throws if called when bigIntEnabled is false. */ + const fI64Disabled = function(fname){ + return ()=>toss(fname+"() disabled due to lack", + "of BigInt support in this build."); + }; + for(const e of wasm.bindingSignatures.int64){ + capi[e[0]] = wasm.bigIntEnabled + ? wasm.xWrap.apply(null, e) + : fI64Disabled(e[0]); + } + + if(wasm.exports.sqlite3_wasm_db_error){ + util.sqlite3_wasm_db_error = capi.wasm.xWrap( + 'sqlite3_wasm_db_error', 'int', 'sqlite3*', 'int', 'string' + ); + }else{ + util.sqlite3_wasm_db_error = function(pDb,errCode,msg){ + console.warn("sqlite3_wasm_db_error() is not exported.",arguments); + return errCode; + }; + } + + /** + When registering a VFS and its related components it may be + necessary to ensure that JS keeps a reference to them to keep + them from getting garbage collected. Simply pass each such value + to this function and a reference will be held to it for the life + of the app. + */ + capi.sqlite3_vfs_register.addReference = function f(...args){ + if(!f._) f._ = []; + f._.push(...args); + }; + + }/*xWrap() bindings*/; + + /** + Scope-local holder of the two impls of sqlite3_prepare_v2/v3(). + */ + const __prepare = Object.create(null); + /** + This binding expects a JS string as its 2nd argument and + null as its final argument. In order to compile multiple + statements from a single string, the "full" impl (see + below) must be used. + */ + __prepare.basic = wasm.xWrap('sqlite3_prepare_v3', + "int", ["sqlite3*", "string", + "int"/*MUST always be negative*/, + "int", "**", + "**"/*MUST be 0 or null or undefined!*/]); + /** + Impl which requires that the 2nd argument be a pointer + to the SQL string, instead of being converted to a + string. This variant is necessary for cases where we + require a non-NULL value for the final argument + (exec()'ing multiple statements from one input + string). For simpler cases, where only the first + statement in the SQL string is required, the wrapper + named sqlite3_prepare_v2() is sufficient and easier to + use because it doesn't require dealing with pointers. + */ + __prepare.full = wasm.xWrap('sqlite3_prepare_v3', + "int", ["sqlite3*", "*", "int", "int", + "**", "**"]); + + /* Documented in the api object's initializer. */ + capi.sqlite3_prepare_v3 = function f(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail){ + /* 2022-07-08: xWrap() 'string' arg handling may be able do this + special-case handling for us. It needs to be tested. Or maybe + not: we always want to treat pzTail as null when passed a + non-pointer SQL string and the argument adapters don't have + enough state to know that. Maybe they could/should, by passing + the currently-collected args as an array as the 2nd arg to the + argument adapters? Or maybe we collect all args in an array, + pass that to an optional post-args-collected callback, and give + it a chance to manipulate the args before we pass them on? */ + if(util.isSQLableTypedArray(sql)) sql = util.typedArrayToString(sql); + switch(typeof sql){ + case 'string': return __prepare.basic(pDb, sql, -1, prepFlags, ppStmt, null); + case 'number': return __prepare.full(pDb, sql, sqlLen||-1, prepFlags, ppStmt, pzTail); + default: + return util.sqlite3_wasm_db_error( + pDb, capi.SQLITE_MISUSE, + "Invalid SQL argument type for sqlite3_prepare_v2/v3()." + ); + } + }; + + capi.sqlite3_prepare_v2 = + (pDb, sql, sqlLen, ppStmt, pzTail)=>capi.sqlite3_prepare_v3(pDb, sql, sqlLen, 0, ppStmt, pzTail); + + /** + Install JS<->C struct bindings for the non-opaque struct types we + need... */ + sqlite3.StructBinder = self.Jaccwabyt({ + heap: 0 ? wasm.memory : wasm.heap8u, + alloc: wasm.alloc, + dealloc: wasm.dealloc, + functionTable: wasm.functionTable, + bigIntEnabled: wasm.bigIntEnabled, + memberPrefix: '$' + }); + delete self.Jaccwabyt; + + {/* Import C-level constants and structs... */ + const cJson = wasm.xCall('sqlite3_wasm_enum_json'); + if(!cJson){ + toss("Maintenance required: increase sqlite3_wasm_enum_json()'s", + "static buffer size!"); + } + wasm.ctype = JSON.parse(wasm.cstringToJs(cJson)); + //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); + for(const t of ['access', 'blobFinalizers', 'dataTypes', + 'encodings', 'flock', 'ioCap', + 'openFlags', 'prepareFlags', 'resultCodes', + 'syncFlags', 'udfFlags', 'version' + ]){ + for(const [k,v] of Object.entries(wasm.ctype[t])){ + capi[k] = v; + } + } + /* Bind all registered C-side structs... */ + for(const s of wasm.ctype.structs){ + capi[s.name] = sqlite3.StructBinder(s); + } + } + +})(self); diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js new file mode 100644 index 0000000000..9e54733966 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -0,0 +1,1438 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file contains the so-called OO #1 API wrapper for the sqlite3 + WASM build. It requires that sqlite3-api-glue.js has already run + and it installs its deliverable as self.sqlite3.oo1. +*/ +(function(self){ + const toss = (...args)=>{throw new Error(args.join(' '))}; + + const sqlite3 = self.sqlite3 || toss("Missing main sqlite3 object."); + const capi = sqlite3.capi, util = capi.util; + /* What follows is colloquially known as "OO API #1". It is a + binding of the sqlite3 API which is designed to be run within + the same thread (main or worker) as the one in which the + sqlite3 WASM binding was initialized. This wrapper cannot use + the sqlite3 binding if, e.g., the wrapper is in the main thread + and the sqlite3 API is in a worker. */ + + /** + In order to keep clients from manipulating, perhaps + inadvertently, the underlying pointer values of DB and Stmt + instances, we'll gate access to them via the `pointer` property + accessor and store their real values in this map. Keys = DB/Stmt + objects, values = pointer values. This also unifies how those are + accessed, for potential use downstream via custom + capi.wasm.xWrap() function signatures which know how to extract + it. + */ + const __ptrMap = new WeakMap(); + /** + Map of DB instances to objects, each object being a map of UDF + names to wasm function _pointers_ added to that DB handle via + createFunction(). + */ + const __udfMap = new WeakMap(); + /** + Map of DB instances to objects, each object being a map of Stmt + wasm pointers to Stmt objects. + */ + const __stmtMap = new WeakMap(); + + /** If object opts has _its own_ property named p then that + property's value is returned, else dflt is returned. */ + const getOwnOption = (opts, p, dflt)=> + opts.hasOwnProperty(p) ? opts[p] : dflt; + + /** + An Error subclass specifically for reporting DB-level errors and + enabling clients to unambiguously identify such exceptions. + */ + class SQLite3Error extends Error { + constructor(...args){ + super(...args); + this.name = 'SQLite3Error'; + } + }; + const toss3 = (...args)=>{throw new SQLite3Error(args)}; + sqlite3.SQLite3Error = SQLite3Error; + + /** + The DB class provides a high-level OO wrapper around an sqlite3 + db handle. + + The given db filename must be resolvable using whatever + filesystem layer (virtual or otherwise) is set up for the default + sqlite3 VFS. + + Note that the special sqlite3 db names ":memory:" and "" + (temporary db) have their normal special meanings here and need + not resolve to real filenames, but "" uses an on-storage + temporary database and requires that the VFS support that. + + The db is currently opened with a fixed set of flags: + (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXRESCODE). This API will change in the future + permit the caller to provide those flags via an additional + argument. + + For purposes of passing a DB instance to C-style sqlite3 + functions, its read-only `pointer` property holds its `sqlite3*` + pointer value. That property can also be used to check whether + this DB instance is still open. + */ + const DB = function ctor(fn=':memory:'){ + if('string'!==typeof fn){ + toss3("Invalid filename for DB constructor."); + } + const stack = capi.wasm.scopedAllocPush(); + let ptr; + try { + const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */; + const rc = capi.sqlite3_open_v2(fn, ppDb, capi.SQLITE_OPEN_READWRITE + | capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_EXRESCODE, null); + ptr = capi.wasm.getMemValue(ppDb, '*'); + ctor.checkRc(ptr, rc); + }catch(e){ + if(ptr) capi.sqlite3_close_v2(ptr); + throw e; + } + finally{capi.wasm.scopedAllocPop(stack);} + this.filename = fn; + __ptrMap.set(this, ptr); + __stmtMap.set(this, Object.create(null)); + __udfMap.set(this, Object.create(null)); + }; + + /** + Internal-use enum for mapping JS types to DB-bindable types. + These do not (and need not) line up with the SQLITE_type + values. All values in this enum must be truthy and distinct + but they need not be numbers. + */ + const BindTypes = { + null: 1, + number: 2, + string: 3, + boolean: 4, + blob: 5 + }; + BindTypes['undefined'] == BindTypes.null; + if(capi.wasm.bigIntEnabled){ + BindTypes.bigint = BindTypes.number; + } + + /** + This class wraps sqlite3_stmt. Calling this constructor + directly will trigger an exception. Use DB.prepare() to create + new instances. + + For purposes of passing a Stmt instance to C-style sqlite3 + functions, its read-only `pointer` property holds its `sqlite3_stmt*` + pointer value. + */ + const Stmt = function(){ + if(BindTypes!==arguments[2]){ + toss3("Do not call the Stmt constructor directly. Use DB.prepare()."); + } + this.db = arguments[0]; + __ptrMap.set(this, arguments[1]); + this.columnCount = capi.sqlite3_column_count(this.pointer); + this.parameterCount = capi.sqlite3_bind_parameter_count(this.pointer); + }; + + /** Throws if the given DB has been closed, else it is returned. */ + const affirmDbOpen = function(db){ + if(!db.pointer) toss3("DB has been closed."); + return db; + }; + + /** Throws if ndx is not an integer or if it is out of range + for stmt.columnCount, else returns stmt. + + Reminder: this will also fail after the statement is finalized + but the resulting error will be about an out-of-bounds column + index. + */ + const affirmColIndex = function(stmt,ndx){ + if((ndx !== (ndx|0)) || ndx<0 || ndx>=stmt.columnCount){ + toss3("Column index",ndx,"is out of range."); + } + return stmt; + }; + + /** + Expects to be passed (arguments) from DB.exec() and + DB.execMulti(). Does the argument processing/validation, throws + on error, and returns a new object on success: + + { sql: the SQL, opt: optionsObj, cbArg: function} + + cbArg is only set if the opt.callback is set, in which case + it's a function which expects to be passed the current Stmt + and returns the callback argument of the type indicated by + the input arguments. + */ + const parseExecArgs = function(args){ + const out = Object.create(null); + out.opt = Object.create(null); + switch(args.length){ + case 1: + if('string'===typeof args[0] || util.isSQLableTypedArray(args[0])){ + out.sql = args[0]; + }else if(args[0] && 'object'===typeof args[0]){ + out.opt = args[0]; + out.sql = out.opt.sql; + } + break; + case 2: + out.sql = args[0]; + out.opt = args[1]; + break; + default: toss3("Invalid argument count for exec()."); + }; + if(util.isSQLableTypedArray(out.sql)){ + out.sql = util.typedArrayToString(out.sql); + }else if(Array.isArray(out.sql)){ + out.sql = out.sql.join(''); + }else if('string'!==typeof out.sql){ + toss3("Missing SQL argument."); + } + if(out.opt.callback || out.opt.resultRows){ + switch((undefined===out.opt.rowMode) + ? 'stmt' : out.opt.rowMode) { + case 'object': out.cbArg = (stmt)=>stmt.get({}); break; + case 'array': out.cbArg = (stmt)=>stmt.get([]); break; + case 'stmt': + if(Array.isArray(out.opt.resultRows)){ + toss3("Invalid rowMode for resultRows array: must", + "be one of 'array', 'object',", + "or a result column number."); + } + out.cbArg = (stmt)=>stmt; + break; + default: + if(util.isInt32(out.opt.rowMode)){ + out.cbArg = (stmt)=>stmt.get(out.opt.rowMode); + break; + } + toss3("Invalid rowMode:",out.opt.rowMode); + } + } + return out; + }; + + /** + Expects to be given a DB instance or an `sqlite3*` pointer, and an + sqlite3 API result code. If the result code is not falsy, this + function throws an SQLite3Error with an error message from + sqlite3_errmsg(), using dbPtr as the db handle. Note that if it's + passed a non-error code like SQLITE_ROW or SQLITE_DONE, it will + still throw but the error string might be "Not an error." The + various non-0 non-error codes need to be checked for in client + code where they are expected. + */ + DB.checkRc = function(dbPtr, sqliteResultCode){ + if(sqliteResultCode){ + if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; + throw new SQLite3Error([ + "sqlite result code",sqliteResultCode+":", + capi.sqlite3_errmsg(dbPtr) || "Unknown db error." + ].join(' ')); + } + }; + + DB.prototype = { + /** + Finalizes all open statements and closes this database + connection. This is a no-op if the db has already been + closed. After calling close(), `this.pointer` will resolve to + `undefined`, so that can be used to check whether the db + instance is still opened. + */ + close: function(){ + if(this.pointer){ + const pDb = this.pointer; + let s; + const that = this; + Object.keys(__stmtMap.get(this)).forEach((k,s)=>{ + if(s && s.pointer) s.finalize(); + }); + Object.values(__udfMap.get(this)).forEach( + capi.wasm.uninstallFunction.bind(capi.wasm) + ); + __ptrMap.delete(this); + __stmtMap.delete(this); + __udfMap.delete(this); + capi.sqlite3_close_v2(pDb); + delete this.filename; + } + }, + /** + Returns the number of changes, as per sqlite3_changes() + (if the first argument is false) or sqlite3_total_changes() + (if it's true). If the 2nd argument is true, it uses + sqlite3_changes64() or sqlite3_total_changes64(), which + will trigger an exception if this build does not have + BigInt support enabled. + */ + changes: function(total=false,sixtyFour=false){ + const p = affirmDbOpen(this).pointer; + if(total){ + return sixtyFour + ? capi.sqlite3_total_changes64(p) + : capi.sqlite3_total_changes(p); + }else{ + return sixtyFour + ? capi.sqlite3_changes64(p) + : capi.sqlite3_changes(p); + } + }, + /** + Similar to this.filename but will return NULL for + special names like ":memory:". Not of much use until + we have filesystem support. Throws if the DB has + been closed. If passed an argument it then it will return + the filename of the ATTACHEd db with that name, else it assumes + a name of `main`. + */ + fileName: function(dbName){ + return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName||"main"); + }, + /** + Returns true if this db instance has a name which resolves to a + file. If the name is "" or ":memory:", it resolves to false. + Note that it is not aware of the peculiarities of URI-style + names and a URI-style name for a ":memory:" db will fool it. + */ + hasFilename: function(){ + const fn = this.filename; + if(!fn || ':memory'===fn) return false; + return true; + }, + /** + Returns the name of the given 0-based db number, as documented + for sqlite3_db_name(). + */ + dbName: function(dbNumber=0){ + return capi.sqlite3_db_name(affirmDbOpen(this).pointer, dbNumber); + }, + /** + Compiles the given SQL and returns a prepared Stmt. This is + the only way to create new Stmt objects. Throws on error. + + The given SQL must be a string, a Uint8Array holding SQL, or a + WASM pointer to memory holding the NUL-terminated SQL string. + If the SQL contains no statements, an SQLite3Error is thrown. + + Design note: the C API permits empty SQL, reporting it as a 0 + result code and a NULL stmt pointer. Supporting that case here + would cause extra work for all clients: any use of the Stmt API + on such a statement will necessarily throw, so clients would be + required to check `stmt.pointer` after calling `prepare()` in + order to determine whether the Stmt instance is empty or not. + Long-time practice (with other sqlite3 script bindings) + suggests that the empty-prepare case is sufficiently rare (and + useless) that supporting it here would simply hurt overall + usability. + */ + prepare: function(sql){ + affirmDbOpen(this); + const stack = capi.wasm.scopedAllocPush(); + let ppStmt, pStmt; + try{ + ppStmt = capi.wasm.scopedAllocPtr()/* output (sqlite3_stmt**) arg */; + DB.checkRc(this, capi.sqlite3_prepare_v2(this.pointer, sql, -1, ppStmt, null)); + pStmt = capi.wasm.getMemValue(ppStmt, '*'); + } + finally {capi.wasm.scopedAllocPop(stack)} + if(!pStmt) toss3("Cannot prepare empty SQL."); + const stmt = new Stmt(this, pStmt, BindTypes); + __stmtMap.get(this)[pStmt] = stmt; + return stmt; + }, + /** + This function works like execMulti(), and takes most of the + same arguments, but is more efficient (performs much less + work) when the input SQL is only a single statement. If + passed a multi-statement SQL, it only processes the first + one. + + This function supports the following additional options not + supported by execMulti(): + + - .multi: if true, this function acts as a proxy for + execMulti() and behaves identically to that function. + + - .columnNames: if this is an array and the query has + result columns, the array is passed to + Stmt.getColumnNames() to append the column names to it + (regardless of whether the query produces any result + rows). If the query has no result columns, this value is + unchanged. + + The following options to execMulti() are _not_ supported by + this method (they are simply ignored): + + - .saveSql + */ + exec: function(/*(sql [,optionsObj]) or (optionsObj)*/){ + affirmDbOpen(this); + const arg = parseExecArgs(arguments); + if(!arg.sql) return this; + else if(arg.opt.multi){ + return this.execMulti(arg, undefined, BindTypes); + } + const opt = arg.opt; + let stmt, rowTarget; + try { + if(Array.isArray(opt.resultRows)){ + rowTarget = opt.resultRows; + } + stmt = this.prepare(arg.sql); + if(stmt.columnCount && Array.isArray(opt.columnNames)){ + stmt.getColumnNames(opt.columnNames); + } + if(opt.bind) stmt.bind(opt.bind); + if(opt.callback || rowTarget){ + while(stmt.step()){ + const row = arg.cbArg(stmt); + if(rowTarget) rowTarget.push(row); + if(opt.callback){ + stmt._isLocked = true; + opt.callback(row, stmt); + stmt._isLocked = false; + } + } + }else{ + stmt.step(); + } + }finally{ + if(stmt){ + delete stmt._isLocked; + stmt.finalize(); + } + } + return this; + }/*exec()*/, + /** + Executes one or more SQL statements in the form of a single + string. Its arguments must be either (sql,optionsObject) or + (optionsObject). In the latter case, optionsObject.sql + must contain the SQL to execute. Returns this + object. Throws on error. + + If no SQL is provided, or a non-string is provided, an + exception is triggered. Empty SQL, on the other hand, is + simply a no-op. + + The optional options object may contain any of the following + properties: + + - .sql = the SQL to run (unless it's provided as the first + argument). This must be of type string, Uint8Array, or an + array of strings (in which case they're concatenated + together as-is, with no separator between elements, + before evaluation). + + - .bind = a single value valid as an argument for + Stmt.bind(). This is ONLY applied to the FIRST non-empty + statement in the SQL which has any bindable + parameters. (Empty statements are skipped entirely.) + + - .callback = a function which gets called for each row of + the FIRST statement in the SQL which has result + _columns_, but only if that statement has any result + _rows_. The second argument passed to the callback is + always the current Stmt object (so that the caller may + collect column names, or similar). The first argument + passed to the callback defaults to the current Stmt + object but may be changed with ... + + - .rowMode = either a string describing what type of argument + should be passed as the first argument to the callback or an + integer representing a result column index. A `rowMode` of + 'object' causes the results of `stmt.get({})` to be passed to + the `callback` and/or appended to `resultRows`. A value of + 'array' causes the results of `stmt.get([])` to be passed to + passed on. A value of 'stmt' is equivalent to the default, + passing the current Stmt to the callback (noting that it's + always passed as the 2nd argument), but this mode will trigger + an exception if `resultRows` is an array. If `rowMode` is an + integer, only the single value from that result column will be + passed on. Any other value for the option triggers an + exception. + + - .resultRows: if this is an array, it functions similarly to + the `callback` option: each row of the result set (if any) of + the FIRST first statement which has result _columns_ is + appended to the array in the format specified for the `rowMode` + option, with the exception that the only legal values for + `rowMode` in this case are 'array' or 'object', neither of + which is the default. It is legal to use both `resultRows` and + `callback`, but `resultRows` is likely much simpler to use for + small data sets and can be used over a WebWorker-style message + interface. execMulti() throws if `resultRows` is set and + `rowMode` is 'stmt' (which is the default!). + + - saveSql = an optional array. If set, the SQL of each + executed statement is appended to this array before the + statement is executed (but after it is prepared - we + don't have the string until after that). Empty SQL + statements are elided. + + See also the exec() method, which is a close cousin of this + one. + + ACHTUNG #1: The callback MUST NOT modify the Stmt + object. Calling any of the Stmt.get() variants, + Stmt.getColumnName(), or similar, is legal, but calling + step() or finalize() is not. Routines which are illegal + in this context will trigger an exception. + + ACHTUNG #2: The semantics of the `bind` and `callback` + options may well change or those options may be removed + altogether for this function (but retained for exec()). + Generally speaking, neither bind parameters nor a callback + are generically useful when executing multi-statement SQL. + */ + execMulti: function(/*(sql [,obj]) || (obj)*/){ + affirmDbOpen(this); + const wasm = capi.wasm; + const arg = (BindTypes===arguments[2] + /* ^^^ Being passed on from exec() */ + ? arguments[0] : parseExecArgs(arguments)); + if(!arg.sql) return this; + const opt = arg.opt; + const callback = opt.callback; + const resultRows = (Array.isArray(opt.resultRows) + ? opt.resultRows : undefined); + if(resultRows && 'stmt'===opt.rowMode){ + toss3("rowMode 'stmt' is not valid in combination", + "with a resultRows array."); + } + let rowMode = (((callback||resultRows) && (undefined!==opt.rowMode)) + ? opt.rowMode : undefined); + let stmt; + let bind = opt.bind; + const stack = wasm.scopedAllocPush(); + try{ + const isTA = util.isSQLableTypedArray(arg.sql) + /* Optimization: if the SQL is a TypedArray we can save some string + conversion costs. */; + /* Allocate the two output pointers (ppStmt, pzTail) and heap + space for the SQL (pSql). When prepare_v2() returns, pzTail + will point to somewhere in pSql. */ + let sqlByteLen = isTA ? arg.sql.byteLength : wasm.jstrlen(arg.sql); + const ppStmt = wasm.scopedAlloc(/* output (sqlite3_stmt**) arg and pzTail */ + (2 * wasm.ptrSizeof) + + (sqlByteLen + 1/* SQL + NUL */)); + const pzTail = ppStmt + wasm.ptrSizeof /* final arg to sqlite3_prepare_v2() */; + let pSql = pzTail + wasm.ptrSizeof; + const pSqlEnd = pSql + sqlByteLen; + if(isTA) wasm.heap8().set(arg.sql, pSql); + else wasm.jstrcpy(arg.sql, wasm.heap8(), pSql, sqlByteLen, false); + wasm.setMemValue(pSql + sqlByteLen, 0/*NUL terminator*/); + while(wasm.getMemValue(pSql, 'i8') + /* Maintenance reminder: ^^^^ _must_ be i8 or else we + will very likely cause an endless loop. What that's + doing is checking for a terminating NUL byte. If we + use i32 or similar then we read 4 bytes, read stuff + around the NUL terminator, and get stuck in and + endless loop at the end of the SQL, endlessly + re-preparing an empty statement. */ ){ + wasm.setMemValue(ppStmt, 0, wasm.ptrIR); + wasm.setMemValue(pzTail, 0, wasm.ptrIR); + DB.checkRc(this, capi.sqlite3_prepare_v2( + this.pointer, pSql, sqlByteLen, ppStmt, pzTail + )); + const pStmt = wasm.getMemValue(ppStmt, wasm.ptrIR); + pSql = wasm.getMemValue(pzTail, wasm.ptrIR); + sqlByteLen = pSqlEnd - pSql; + if(!pStmt) continue; + if(Array.isArray(opt.saveSql)){ + opt.saveSql.push(capi.sqlite3_sql(pStmt).trim()); + } + stmt = new Stmt(this, pStmt, BindTypes); + if(bind && stmt.parameterCount){ + stmt.bind(bind); + bind = null; + } + if(stmt.columnCount && undefined!==rowMode){ + /* Only forward SELECT results for the FIRST query + in the SQL which potentially has them. */ + while(stmt.step()){ + stmt._isLocked = true; + const row = arg.cbArg(stmt); + if(callback) callback(row, stmt); + if(resultRows) resultRows.push(row); + stmt._isLocked = false; + } + rowMode = undefined; + }else{ + // Do we need to while(stmt.step()){} here? + stmt.step(); + } + stmt.finalize(); + stmt = null; + } + }catch(e){ + console.warn("DB.execMulti() is propagating exception",opt,e); + throw e; + }finally{ + if(stmt){ + delete stmt._isLocked; + stmt.finalize(); + } + wasm.scopedAllocPop(stack); + } + return this; + }/*execMulti()*/, + /** + Creates a new scalar UDF (User-Defined Function) which is + accessible via SQL code. This function may be called in any + of the following forms: + + - (name, function) + - (name, function, optionsObject) + - (name, optionsObject) + - (optionsObject) + + In the final two cases, the function must be defined as the + 'callback' property of the options object. In the final + case, the function's name must be the 'name' property. + + This can only be used to create scalar functions, not + aggregate or window functions. UDFs cannot be removed from + a DB handle after they're added. + + On success, returns this object. Throws on error. + + When called from SQL, arguments to the UDF, and its result, + will be converted between JS and SQL with as much fidelity + as is feasible, triggering an exception if a type + conversion cannot be determined. Some freedom is afforded + to numeric conversions due to friction between the JS and C + worlds: integers which are larger than 32 bits will be + treated as doubles, as JS does not support 64-bit integers + and it is (as of this writing) illegal to use WASM + functions which take or return 64-bit integers from JS. + + The optional options object may contain flags to modify how + the function is defined: + + - .arity: the number of arguments which SQL calls to this + function expect or require. The default value is the + callback's length property (i.e. the number of declared + parameters it has). A value of -1 means that the function + is variadic and may accept any number of arguments, up to + sqlite3's compile-time limits. sqlite3 will enforce the + argument count if is zero or greater. + + The following properties correspond to flags documented at: + + https://sqlite.org/c3ref/create_function.html + + - .deterministic = SQLITE_DETERMINISTIC + - .directOnly = SQLITE_DIRECTONLY + - .innocuous = SQLITE_INNOCUOUS + + Maintenance reminder: the ability to add new + WASM-accessible functions to the runtime requires that the + WASM build is compiled with emcc's `-sALLOW_TABLE_GROWTH` + flag. + */ + createFunction: function f(name, callback,opt){ + switch(arguments.length){ + case 1: /* (optionsObject) */ + opt = name; + name = opt.name; + callback = opt.callback; + break; + case 2: /* (name, callback|optionsObject) */ + if(!(callback instanceof Function)){ + opt = callback; + callback = opt.callback; + } + break; + default: break; + } + if(!opt) opt = {}; + if(!(callback instanceof Function)){ + toss3("Invalid arguments: expecting a callback function."); + }else if('string' !== typeof name){ + toss3("Invalid arguments: missing function name."); + } + if(!f._extractArgs){ + /* Static init */ + f._extractArgs = function(argc, pArgv){ + let i, pVal, valType, arg; + const tgt = []; + for(i = 0; i < argc; ++i){ + pVal = capi.wasm.getMemValue(pArgv + (capi.wasm.ptrSizeof * i), + capi.wasm.ptrIR); + /** + Curiously: despite ostensibly requiring 8-byte + alignment, the pArgv array is parcelled into chunks of + 4 bytes (1 pointer each). The values those point to + have 8-byte alignment but the individual argv entries + do not. + */ + valType = capi.sqlite3_value_type(pVal); + switch(valType){ + case capi.SQLITE_INTEGER: + case capi.SQLITE_FLOAT: + arg = capi.sqlite3_value_double(pVal); + break; + case capi.SQLITE_TEXT: + arg = capi.sqlite3_value_text(pVal); + break; + case capi.SQLITE_BLOB:{ + const n = capi.sqlite3_value_bytes(pVal); + const pBlob = capi.sqlite3_value_blob(pVal); + arg = new Uint8Array(n); + let i; + const heap = n ? capi.wasm.heap8() : false; + for(i = 0; i < n; ++i) arg[i] = heap[pBlob+i]; + break; + } + case capi.SQLITE_NULL: + arg = null; break; + default: + toss3("Unhandled sqlite3_value_type()",valType, + "is possibly indicative of incorrect", + "pointer size assumption."); + } + tgt.push(arg); + } + return tgt; + }/*_extractArgs()*/; + f._setResult = function(pCx, val){ + switch(typeof val) { + case 'boolean': + capi.sqlite3_result_int(pCx, val ? 1 : 0); + break; + case 'number': { + (util.isInt32(val) + ? capi.sqlite3_result_int + : capi.sqlite3_result_double)(pCx, val); + break; + } + case 'string': + capi.sqlite3_result_text(pCx, val, -1, capi.SQLITE_TRANSIENT); + break; + case 'object': + if(null===val) { + capi.sqlite3_result_null(pCx); + break; + }else if(util.isBindableTypedArray(val)){ + const pBlob = capi.wasm.mallocFromTypedArray(val); + capi.sqlite3_result_blob(pCx, pBlob, val.byteLength, + capi.SQLITE_TRANSIENT); + capi.wasm.dealloc(pBlob); + break; + } + // else fall through + default: + toss3("Don't not how to handle this UDF result value:",val); + }; + }/*_setResult()*/; + }/*static init*/ + const wrapper = function(pCx, argc, pArgv){ + try{ + f._setResult(pCx, callback.apply(null, f._extractArgs(argc, pArgv))); + }catch(e){ + if(e instanceof capi.WasmAllocError){ + capi.sqlite3_result_error_nomem(pCx); + }else{ + capi.sqlite3_result_error(pCx, e.message, -1); + } + } + }; + const pUdf = capi.wasm.installFunction(wrapper, "v(iii)"); + let fFlags = 0 /*flags for sqlite3_create_function_v2()*/; + if(getOwnOption(opt, 'deterministic')) fFlags |= capi.SQLITE_DETERMINISTIC; + if(getOwnOption(opt, 'directOnly')) fFlags |= capi.SQLITE_DIRECTONLY; + if(getOwnOption(opt, 'innocuous')) fFlags |= capi.SQLITE_INNOCUOUS; + name = name.toLowerCase(); + try { + DB.checkRc(this, capi.sqlite3_create_function_v2( + this.pointer, name, + (opt.hasOwnProperty('arity') ? +opt.arity : callback.length), + capi.SQLITE_UTF8 | fFlags, null/*pApp*/, pUdf, + null/*xStep*/, null/*xFinal*/, null/*xDestroy*/)); + }catch(e){ + capi.wasm.uninstallFunction(pUdf); + throw e; + } + const udfMap = __udfMap.get(this); + if(udfMap[name]){ + try{capi.wasm.uninstallFunction(udfMap[name])} + catch(e){/*ignore*/} + } + udfMap[name] = pUdf; + return this; + }/*createFunction()*/, + /** + Prepares the given SQL, step()s it one time, and returns + the value of the first result column. If it has no results, + undefined is returned. + + If passed a second argument, it is treated like an argument + to Stmt.bind(), so may be any type supported by that + function. Passing the undefined value is the same as passing + no value, which is useful when... + + If passed a 3rd argument, it is expected to be one of the + SQLITE_{typename} constants. Passing the undefined value is + the same as not passing a value. + + Throws on error (e.g. malformedSQL). + */ + selectValue: function(sql,bind,asType){ + let stmt, rc; + try { + stmt = this.prepare(sql).bind(bind); + if(stmt.step()) rc = stmt.get(0,asType); + }finally{ + if(stmt) stmt.finalize(); + } + return rc; + }, + + /** + Returns the number of currently-opened Stmt handles for this db + handle, or 0 if this DB instance is closed. + */ + openStatementCount: function(){ + return this.pointer ? Object.keys(__stmtMap.get(this)).length : 0; + }, + + /** + This function currently does nothing and always throws. It + WILL BE REMOVED pending other refactoring, to eliminate a hard + dependency on Emscripten. This feature will be moved into a + higher-level API or a runtime-configurable feature. + + That said, what its replacement should eventually do is... + + Exports a copy of this db's file as a Uint8Array and + returns it. It is technically not legal to call this while + any prepared statement are currently active because, + depending on the platform, it might not be legal to read + the db while a statement is locking it. Throws if this db + is not open or has any opened statements. + + The resulting buffer can be passed to this class's + constructor to restore the DB. + + Maintenance reminder: the corresponding sql.js impl of this + feature closes the current db, finalizing any active + statements and (seemingly unnecessarily) destroys any UDFs, + copies the file, and then re-opens it (without restoring + the UDFs). Those gymnastics are not necessary on the tested + platform but might be necessary on others. Because of that + eventuality, this interface currently enforces that no + statements are active when this is run. It will throw if + any are. + */ + exportBinaryImage: function(){ + toss3("exportBinaryImage() is slated for removal for portability reasons."); + /*********************** + The following is currently kept only for reference when + porting to some other layer, noting that we may well not be + able to implement this, at this level, when using the OPFS + VFS because of its exclusive locking policy. + + affirmDbOpen(this); + if(this.openStatementCount()>0){ + toss3("Cannot export with prepared statements active!", + "finalize() all statements and try again."); + } + return MODCFG.FS.readFile(this.filename, {encoding:"binary"}); + ***********************/ + } + }/*DB.prototype*/; + + + /** Throws if the given Stmt has been finalized, else stmt is + returned. */ + const affirmStmtOpen = function(stmt){ + if(!stmt.pointer) toss3("Stmt has been closed."); + return stmt; + }; + + /** Returns an opaque truthy value from the BindTypes + enum if v's type is a valid bindable type, else + returns a falsy value. As a special case, a value of + undefined is treated as a bind type of null. */ + const isSupportedBindType = function(v){ + let t = BindTypes[(null===v||undefined===v) ? 'null' : typeof v]; + switch(t){ + case BindTypes.boolean: + case BindTypes.null: + case BindTypes.number: + case BindTypes.string: + return t; + case BindTypes.bigint: + if(capi.wasm.bigIntEnabled) return t; + /* else fall through */ + default: + //console.log("isSupportedBindType",t,v); + return util.isBindableTypedArray(v) ? BindTypes.blob : undefined; + } + }; + + /** + If isSupportedBindType(v) returns a truthy value, this + function returns that value, else it throws. + */ + const affirmSupportedBindType = function(v){ + //console.log('affirmSupportedBindType',v); + return isSupportedBindType(v) || toss3("Unsupported bind() argument type:",typeof v); + }; + + /** + If key is a number and within range of stmt's bound parameter + count, key is returned. + + If key is not a number then it is checked against named + parameters. If a match is found, its index is returned. + + Else it throws. + */ + const affirmParamIndex = function(stmt,key){ + const n = ('number'===typeof key) + ? key : capi.sqlite3_bind_parameter_index(stmt.pointer, key); + if(0===n || !util.isInt32(n)){ + toss3("Invalid bind() parameter name: "+key); + } + else if(n<1 || n>stmt.parameterCount) toss3("Bind index",key,"is out of range."); + return n; + }; + + /** + If stmt._isLocked is truthy, this throws an exception + complaining that the 2nd argument (an operation name, + e.g. "bind()") is not legal while the statement is "locked". + Locking happens before an exec()-like callback is passed a + statement, to ensure that the callback does not mutate or + finalize the statement. If it does not throw, it returns stmt. + */ + const affirmUnlocked = function(stmt,currentOpName){ + if(stmt._isLocked){ + toss3("Operation is illegal when statement is locked:",currentOpName); + } + return stmt; + }; + + /** + Binds a single bound parameter value on the given stmt at the + given index (numeric or named) using the given bindType (see + the BindTypes enum) and value. Throws on error. Returns stmt on + success. + */ + const bindOne = function f(stmt,ndx,bindType,val){ + affirmUnlocked(stmt, 'bind()'); + if(!f._){ + if(capi.wasm.bigIntEnabled){ + f._maxInt = BigInt("0x7fffffffffffffff"); + f._minInt = ~f._maxInt; + } + /* Reminder: when not in BigInt mode, it's impossible for + JS to represent a number out of the range we can bind, + so we have no range checking. */ + f._ = { + string: function(stmt, ndx, val, asBlob){ + if(1){ + /* _Hypothetically_ more efficient than the impl in the 'else' block. */ + const stack = capi.wasm.scopedAllocPush(); + try{ + const n = capi.wasm.jstrlen(val); + const pStr = capi.wasm.scopedAlloc(n); + capi.wasm.jstrcpy(val, capi.wasm.heap8u(), pStr, n, false); + const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; + return f(stmt.pointer, ndx, pStr, n, capi.SQLITE_TRANSIENT); + }finally{ + capi.wasm.scopedAllocPop(stack); + } + }else{ + const bytes = capi.wasm.jstrToUintArray(val,false); + const pStr = capi.wasm.alloc(bytes.length || 1); + capi.wasm.heap8u().set(bytes.length ? bytes : [0], pStr); + try{ + const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; + return f(stmt.pointer, ndx, pStr, bytes.length, capi.SQLITE_TRANSIENT); + }finally{ + capi.wasm.dealloc(pStr); + } + } + } + }; + } + affirmSupportedBindType(val); + ndx = affirmParamIndex(stmt,ndx); + let rc = 0; + switch((null===val || undefined===val) ? BindTypes.null : bindType){ + case BindTypes.null: + rc = capi.sqlite3_bind_null(stmt.pointer, ndx); + break; + case BindTypes.string: + rc = f._.string(stmt, ndx, val, false); + break; + case BindTypes.number: { + let m; + if(util.isInt32(val)) m = capi.sqlite3_bind_int; + else if(capi.wasm.bigIntEnabled && ('bigint'===typeof val)){ + if(valf._maxInt){ + toss3("BigInt value is out of range for int64: "+val); + } + m = capi.sqlite3_bind_int64; + }else if(Number.isInteger(val)){ + m = capi.sqlite3_bind_int64; + }else{ + m = capi.sqlite3_bind_double; + } + rc = m(stmt.pointer, ndx, val); + break; + } + case BindTypes.boolean: + rc = capi.sqlite3_bind_int(stmt.pointer, ndx, val ? 1 : 0); + break; + case BindTypes.blob: { + if('string'===typeof val){ + rc = f._.string(stmt, ndx, val, true); + }else if(!util.isBindableTypedArray(val)){ + toss3("Binding a value as a blob requires", + "that it be a string, Uint8Array, or Int8Array."); + }else if(1){ + /* _Hypothetically_ more efficient than the impl in the 'else' block. */ + const stack = capi.wasm.scopedAllocPush(); + try{ + const pBlob = capi.wasm.scopedAlloc(val.byteLength || 1); + capi.wasm.heap8().set(val.byteLength ? val : [0], pBlob) + rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, + capi.SQLITE_TRANSIENT); + }finally{ + capi.wasm.scopedAllocPop(stack); + } + }else{ + const pBlob = capi.wasm.mallocFromTypedArray(val); + try{ + rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, + capi.SQLITE_TRANSIENT); + }finally{ + capi.wasm.dealloc(pBlob); + } + } + break; + } + default: + console.warn("Unsupported bind() argument type:",val); + toss3("Unsupported bind() argument type: "+(typeof val)); + } + if(rc) checkDbRc(stmt.db.pointer, rc); + return stmt; + }; + + Stmt.prototype = { + /** + "Finalizes" this statement. This is a no-op if the + statement has already been finalizes. Returns + undefined. Most methods in this class will throw if called + after this is. + */ + finalize: function(){ + if(this.pointer){ + affirmUnlocked(this,'finalize()'); + delete __stmtMap.get(this.db)[this.pointer]; + capi.sqlite3_finalize(this.pointer); + __ptrMap.delete(this); + delete this.columnCount; + delete this.parameterCount; + delete this.db; + delete this._isLocked; + } + }, + /** Clears all bound values. Returns this object. + Throws if this statement has been finalized. */ + clearBindings: function(){ + affirmUnlocked(affirmStmtOpen(this), 'clearBindings()') + capi.sqlite3_clear_bindings(this.pointer); + this._mayGet = false; + return this; + }, + /** + Resets this statement so that it may be step()ed again + from the beginning. Returns this object. Throws if this + statement has been finalized. + + If passed a truthy argument then this.clearBindings() is + also called, otherwise any existing bindings, along with + any memory allocated for them, are retained. + */ + reset: function(alsoClearBinds){ + affirmUnlocked(this,'reset()'); + if(alsoClearBinds) this.clearBindings(); + capi.sqlite3_reset(affirmStmtOpen(this).pointer); + this._mayGet = false; + return this; + }, + /** + Binds one or more values to its bindable parameters. It + accepts 1 or 2 arguments: + + If passed a single argument, it must be either an array, an + object, or a value of a bindable type (see below). + + If passed 2 arguments, the first one is the 1-based bind + index or bindable parameter name and the second one must be + a value of a bindable type. + + Bindable value types: + + - null is bound as NULL. + + - undefined as a standalone value is a no-op intended to + simplify certain client-side use cases: passing undefined + as a value to this function will not actually bind + anything and this function will skip confirmation that + binding is even legal. (Those semantics simplify certain + client-side uses.) Conversely, a value of undefined as an + array or object property when binding an array/object + (see below) is treated the same as null. + + - Numbers are bound as either doubles or integers: doubles + if they are larger than 32 bits, else double or int32, + depending on whether they have a fractional part. (It is, + as of this writing, illegal to call (from JS) a WASM + function which either takes or returns an int64.) + Booleans are bound as integer 0 or 1. It is not expected + the distinction of binding doubles which have no + fractional parts is integers is significant for the + majority of clients due to sqlite3's data typing + model. If capi.wasm.bigIntEnabled is true then this + routine will bind BigInt values as 64-bit integers. + + - Strings are bound as strings (use bindAsBlob() to force + blob binding). + + - Uint8Array and Int8Array instances are bound as blobs. + (TODO: binding the other TypedArray types.) + + If passed an array, each element of the array is bound at + the parameter index equal to the array index plus 1 + (because arrays are 0-based but binding is 1-based). + + If passed an object, each object key is treated as a + bindable parameter name. The object keys _must_ match any + bindable parameter names, including any `$`, `@`, or `:` + prefix. Because `$` is a legal identifier chararacter in + JavaScript, that is the suggested prefix for bindable + parameters: `stmt.bind({$a: 1, $b: 2})`. + + It returns this object on success and throws on + error. Errors include: + + - Any bind index is out of range, a named bind parameter + does not match, or this statement has no bindable + parameters. + + - Any value to bind is of an unsupported type. + + - Passed no arguments or more than two. + + - The statement has been finalized. + */ + bind: function(/*[ndx,] arg*/){ + affirmStmtOpen(this); + let ndx, arg; + switch(arguments.length){ + case 1: ndx = 1; arg = arguments[0]; break; + case 2: ndx = arguments[0]; arg = arguments[1]; break; + default: toss3("Invalid bind() arguments."); + } + if(undefined===arg){ + /* It might seem intuitive to bind undefined as NULL + but this approach simplifies certain client-side + uses when passing on arguments between 2+ levels of + functions. */ + return this; + }else if(!this.parameterCount){ + toss3("This statement has no bindable parameters."); + } + this._mayGet = false; + if(null===arg){ + /* bind NULL */ + return bindOne(this, ndx, BindTypes.null, arg); + } + else if(Array.isArray(arg)){ + /* bind each entry by index */ + if(1!==arguments.length){ + toss3("When binding an array, an index argument is not permitted."); + } + arg.forEach((v,i)=>bindOne(this, i+1, affirmSupportedBindType(v), v)); + return this; + } + else if('object'===typeof arg/*null was checked above*/ + && !util.isBindableTypedArray(arg)){ + /* Treat each property of arg as a named bound parameter. */ + if(1!==arguments.length){ + toss3("When binding an object, an index argument is not permitted."); + } + Object.keys(arg) + .forEach(k=>bindOne(this, k, + affirmSupportedBindType(arg[k]), + arg[k])); + return this; + }else{ + return bindOne(this, ndx, affirmSupportedBindType(arg), arg); + } + toss3("Should not reach this point."); + }, + /** + Special case of bind() which binds the given value using the + BLOB binding mechanism instead of the default selected one for + the value. The ndx may be a numbered or named bind index. The + value must be of type string, null/undefined (both get treated + as null), or a TypedArray of a type supported by the bind() + API. + + If passed a single argument, a bind index of 1 is assumed and + the first argument is the value. + */ + bindAsBlob: function(ndx,arg){ + affirmStmtOpen(this); + if(1===arguments.length){ + arg = ndx; + ndx = 1; + } + const t = affirmSupportedBindType(arg); + if(BindTypes.string !== t && BindTypes.blob !== t + && BindTypes.null !== t){ + toss3("Invalid value type for bindAsBlob()"); + } + bindOne(this, ndx, BindTypes.blob, arg); + this._mayGet = false; + return this; + }, + /** + Steps the statement one time. If the result indicates that + a row of data is available, true is returned. If no row of + data is available, false is returned. Throws on error. + */ + step: function(){ + affirmUnlocked(this, 'step()'); + const rc = capi.sqlite3_step(affirmStmtOpen(this).pointer); + switch(rc){ + case capi.SQLITE_DONE: return this._mayGet = false; + case capi.SQLITE_ROW: return this._mayGet = true; + default: + this._mayGet = false; + console.warn("sqlite3_step() rc=",rc,"SQL =", + capi.sqlite3_sql(this.pointer)); + checkDbRc(this.db.pointer, rc); + }; + }, + /** + Fetches the value from the given 0-based column index of + the current data row, throwing if index is out of range. + + Requires that step() has just returned a truthy value, else + an exception is thrown. + + By default it will determine the data type of the result + automatically. If passed a second arugment, it must be one + of the enumeration values for sqlite3 types, which are + defined as members of the sqlite3 module: SQLITE_INTEGER, + SQLITE_FLOAT, SQLITE_TEXT, SQLITE_BLOB. Any other value, + except for undefined, will trigger an exception. Passing + undefined is the same as not passing a value. It is legal + to, e.g., fetch an integer value as a string, in which case + sqlite3 will convert the value to a string. + + If ndx is an array, this function behaves a differently: it + assigns the indexes of the array, from 0 to the number of + result columns, to the values of the corresponding column, + and returns that array. + + If ndx is a plain object, this function behaves even + differentlier: it assigns the properties of the object to + the values of their corresponding result columns. + + Blobs are returned as Uint8Array instances. + + Potential TODO: add type ID SQLITE_JSON, which fetches the + result as a string and passes it (if it's not null) to + JSON.parse(), returning the result of that. Until then, + getJSON() can be used for that. + */ + get: function(ndx,asType){ + if(!affirmStmtOpen(this)._mayGet){ + toss3("Stmt.step() has not (recently) returned true."); + } + if(Array.isArray(ndx)){ + let i = 0; + while(i=Number.MIN_SAFE_INTEGER && rc<=Number.MAX_SAFE_INTEGER){ + /* Coerce "normal" number ranges to normal number values, + and only return BigInt-type values for numbers out of this + range. */ + return Number(rc).valueOf(); + } + return rc; + }else{ + const rc = capi.sqlite3_column_double(this.pointer, ndx); + if(rc>Number.MAX_SAFE_INTEGER || rctoss3("The pointer property is read-only.") + } + Object.defineProperty(Stmt.prototype, 'pointer', prop); + Object.defineProperty(DB.prototype, 'pointer', prop); + } + + /** The OO API's public namespace. */ + sqlite3.oo1 = { + version: { + lib: capi.sqlite3_libversion(), + ooApi: "0.1" + }, + DB, + Stmt + }/*SQLite3 object*/; +})(self); diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js new file mode 100644 index 0000000000..a04029e302 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -0,0 +1,394 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file contains extensions to the sqlite3 WASM API related to the + Origin-Private FileSystem (OPFS). It is intended to be appended to + the main JS deliverable somewhere after sqlite3-api-glue.js and + before sqlite3-api-cleanup.js. + + Significant notes and limitations: + + - As of this writing, OPFS is still very much in flux and only + available in bleeding-edge versions of Chrome (v102+, noting that + that number will increase as the OPFS API matures). + + - The _synchronous_ family of OPFS features (which is what this API + requires) are only available in non-shared Worker threads. This + file tries to detect that case and becomes a no-op if those + features do not seem to be available. +*/ + +// FileSystemHandle +// FileSystemDirectoryHandle +// FileSystemFileHandle +// FileSystemFileHandle.prototype.createSyncAccessHandle +self.sqlite3.postInit.push(function(self, sqlite3){ + const warn = console.warn.bind(console); + if(!self.importScripts || !self.FileSystemFileHandle + || !self.FileSystemFileHandle.prototype.createSyncAccessHandle){ + warn("OPFS not found or its sync API is not available in this environment."); + return; + }else if(!sqlite3.capi.wasm.bigIntEnabled){ + error("OPFS requires BigInt support but sqlite3.capi.wasm.bigIntEnabled is false."); + return; + } + //warn('self.FileSystemFileHandle =',self.FileSystemFileHandle); + //warn('self.FileSystemFileHandle.prototype =',self.FileSystemFileHandle.prototype); + const toss = (...args)=>{throw new Error(args.join(' '))}; + /* This is a web worker, so init the worker-based API. */ + const capi = sqlite3.capi, + wasm = capi.wasm; + const sqlite3_vfs = capi.sqlite3_vfs + || toss("Missing sqlite3.capi.sqlite3_vfs object."); + const sqlite3_file = capi.sqlite3_file + || toss("Missing sqlite3.capi.sqlite3_file object."); + const sqlite3_io_methods = capi.sqlite3_io_methods + || toss("Missing sqlite3.capi.sqlite3_io_methods object."); + const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder."); + const error = console.error.bind(console), + debug = console.debug.bind(console), + log = console.log.bind(console); + warn("UNDER CONSTRUCTION: setting up OPFS VFS..."); + + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. Though we cannot currently handle + that case, the hope is to eventually be able to. */; + const oVfs = new sqlite3_vfs(); + const oIom = new sqlite3_io_methods(); + oVfs.$iVersion = 2/*yes, two*/; + oVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + oVfs.$mxPathname = 1024/*sure, why not?*/; + oVfs.$zName = wasm.allocCString("opfs"); + oVfs.ondispose = [ + '$zName', oVfs.$zName, + 'cleanup dVfs', ()=>(dVfs ? dVfs.dispose() : null) + ]; + if(dVfs){ + oVfs.$xSleep = dVfs.$xSleep; + oVfs.$xRandomness = dVfs.$xRandomness; + } + // All C-side memory of oVfs is zeroed out, but just to be explicit: + oVfs.$xDlOpen = oVfs.$xDlError = oVfs.$xDlSym = oVfs.$xDlClose = null; + + /** + Pedantic sidebar about oVfs.ondispose: the entries in that array + are items to clean up when oVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown order + of the wasm engine and the JS one are undefined so there is no + guaranty that the oVfs instance would be available in one + environment or the other when sqlite3_os_end() is called (_if_ it + gets called at all in a wasm build, which is undefined). + */ + + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructType target object. + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error (e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc.). + + Returns a proxy for this function which is bound to tgt and takes + 2 args (name,func). That function returns the same thing, + permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + */ + const installMethod = function callee(tgt, name, func){ + if(!(tgt instanceof StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt,n,f); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name," is not a function pointer. Signature =",sigN); + } + const memKey = tgt.memberKey(name); + //log("installMethod",tgt, name, sigN); + const fProxy = 1 + // We can remove this proxy middle-man once the VFS is working + ? callee.argcProxy(func, sigN) + : func; + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + if(!tgt.ondispose.__removeFuncList){ + tgt.ondispose.push('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + return (n,f)=>callee(tgt, n, f); + }/*installMethod*/; + + /** + Map of sqlite3_file pointers to OPFS handles. + */ + const __opfsHandles = Object.create(null); + + const randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(''); + }; + + //const rootDir = await navigator.storage.getDirectory(); + + //////////////////////////////////////////////////////////////////////// + // Set up OPFS VFS methods... + let inst = installMethod(oVfs); + inst('xOpen', function(pVfs, zName, pFile, flags, pOutFlags){ + const f = new sqlite3_file(pFile); + f.$pMethods = oIom.pointer; + __opfsHandles[pFile] = f; + f.opfsHandle = null /* TODO */; + if(capi.SQLITE_OPEN_DELETEONCLOSE){ + f.deleteOnClose = true; + } + f.filename = zName ? wasm.cstringToJs(zName) : randomFilename(); + error("OPFS sqlite3_vfs::xOpen is not yet full implemented."); + return capi.SQLITE_IOERR; + }) + ('xFullPathname', function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return i SQLITE_DEFAULT_SECTOR_SIZE */; + //}) + + const rc = capi.sqlite3_vfs_register(oVfs.pointer, 0); + if(rc){ + oVfs.dispose(); + toss("sqlite3_vfs_register(OPFS) failed with rc",rc); + } + capi.sqlite3_vfs_register.addReference(oVfs, oIom); + warn("End of (very incomplete) OPFS setup.", oVfs); + //oVfs.dispose()/*only because we can't yet do anything with it*/; +}); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js new file mode 100644 index 0000000000..60ed61477e --- /dev/null +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -0,0 +1,593 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is intended to be combined at build-time with other + related code, most notably a header and footer which wraps this whole + file into an Emscripten Module.postRun() handler which has a parameter + named "Module" (the Emscripten Module object). The exact requirements, + conventions, and build process are very much under construction and + will be (re)documented once they've stopped fluctuating so much. + + Specific goals of this project: + + - Except where noted in the non-goals, provide a more-or-less + feature-complete wrapper to the sqlite3 C API, insofar as WASM + feature parity with C allows for. In fact, provide at least 3 + APIs... + + 1) Bind a low-level sqlite3 API which is as close to the native + one as feasible in terms of usage. + + 2) A higher-level API, more akin to sql.js and node.js-style + implementations. This one speaks directly to the low-level + API. This API must be used from the same thread as the + low-level API. + + 3) A second higher-level API which speaks to the previous APIs via + worker messages. This one is intended for use in the main + thread, with the lower-level APIs installed in a Worker thread, + and talking to them via Worker messages. Because Workers are + asynchronouns and have only a single message channel, some + acrobatics are needed here to feed async work results back to + the client (as we cannot simply pass around callbacks between + the main and Worker threads). + + - Insofar as possible, support client-side storage using JS + filesystem APIs. As of this writing, such things are still very + much TODO. Initial testing with using IndexedDB as backing storage + showed it to work reasonably well, but it's also too easy to + corrupt by using a web page in two browser tabs because IndexedDB + lacks the locking features needed to support that. + + Specific non-goals of this project: + + - As WASM is a web-centric technology and UTF-8 is the King of + Encodings in that realm, there are no currently plans to support + the UTF16-related sqlite3 APIs. They would add a complication to + the bindings for no appreciable benefit. Though web-related + implementation details take priority, the lower-level WASM module + "should" work in non-web WASM environments. + + - Supporting old or niche-market platforms. WASM is built for a + modern web and requires modern platforms. + + - Though scalar User-Defined Functions (UDFs) may be created in + JavaScript, there are currently no plans to add support for + aggregate and window functions. + + Attribution: + + This project is endebted to the work of sql.js: + + https://github.com/sql-js/sql.js + + sql.js was an essential stepping stone in this code's development as + it demonstrated how to handle some of the WASM-related voodoo (like + handling pointers-to-pointers and adding JS implementations of + C-bound callback functions). These APIs have a considerably + different shape than sql.js's, however. +*/ + +/** + This global symbol is is only a temporary measure: the JS-side + post-processing will remove that object from the global scope when + setup is complete. We require it there temporarily in order to glue + disparate parts together during the loading of the API (which spans + several components). + + This function requires a configuration object intended to abstract + away details specific to any given WASM environment, primarily so + that it can be used without any _direct_ dependency on Emscripten. + (That said, OO API #1 requires, as of this writing, Emscripten's + virtual filesystem API. Baby steps.) +*/ +self.sqlite3ApiBootstrap = function(config){ + 'use strict'; + + /** Throws a new Error, the message of which is the concatenation + all args with a space between each. */ + const toss = (...args)=>{throw new Error(args.join(' '))}; + + /** + Returns true if n is a 32-bit (signed) integer, else + false. This is used for determining when we need to switch to + double-type DB operations for integer values in order to keep + more precision. + */ + const isInt32 = function(n){ + return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/) + && !!(n===(n|0) && n<=2147483647 && n>=-2147483648); + }; + + /** Returns v if v appears to be a TypedArray, else false. */ + const isTypedArray = (v)=>{ + return (v && v.constructor && isInt32(v.constructor.BYTES_PER_ELEMENT)) ? v : false; + }; + + /** + Returns true if v appears to be one of our bind()-able + TypedArray types: Uint8Array or Int8Array. Support for + TypedArrays with element sizes >1 is TODO. + */ + const isBindableTypedArray = (v)=>{ + return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + }; + + /** + Returns true if v appears to be one of the TypedArray types + which is legal for holding SQL code (as opposed to binary blobs). + + Currently this is the same as isBindableTypedArray() but it + seems likely that we'll eventually want to add Uint32Array + and friends to the isBindableTypedArray() list but not to the + isSQLableTypedArray() list. + */ + const isSQLableTypedArray = (v)=>{ + return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + }; + + /** Returns true if isBindableTypedArray(v) does, else throws with a message + that v is not a supported TypedArray value. */ + const affirmBindableTypedArray = (v)=>{ + return isBindableTypedArray(v) + || toss("Value is not of a supported TypedArray type."); + }; + + const utf8Decoder = new TextDecoder('utf-8'); + const typedArrayToString = (str)=>utf8Decoder.decode(str); + + /** + An Error subclass specifically for reporting Wasm-level malloc() + failure and enabling clients to unambiguously identify such + exceptions. + */ + class WasmAllocError extends Error { + constructor(...args){ + super(...args); + this.name = 'WasmAllocError'; + } + }; + + /** + The main sqlite3 binding API gets installed into this object, + mimicking the C API as closely as we can. The numerous members + names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as + possible, identically to the C-native counterparts, as documented at: + + https://www.sqlite.org/c3ref/intro.html + + A very few exceptions require an additional level of proxy + function or may otherwise require special attention in the WASM + environment, and all such cases are document here. Those not + documented here are installed as 1-to-1 proxies for their C-side + counterparts. + */ + const capi = { + /** + An Error subclass which is thrown by this object's alloc() method + on OOM. + */ + WasmAllocError: WasmAllocError, + /** + The API's one single point of access to the WASM-side memory + allocator. Works like malloc(3) (and is likely bound to + malloc()) but throws an WasmAllocError if allocation fails. It is + important that any code which might pass through the sqlite3 C + API NOT throw and must instead return SQLITE_NOMEM (or + equivalent, depending on the context). + + That said, very few cases in the API can result in + client-defined functions propagating exceptions via the C-style + API. Most notably, this applies ot User-defined SQL Functions + (UDFs) registered via sqlite3_create_function_v2(). For that + specific case it is recommended that all UDF creation be + funneled through a utility function and that a wrapper function + be added around the UDF which catches any exception and sets + the error state to OOM. (The overall complexity of registering + UDFs essentially requires a helper for doing so!) + */ + alloc: undefined/*installed later*/, + /** + The API's one single point of access to the WASM-side memory + deallocator. Works like free(3) (and is likely bound to + free()). + */ + dealloc: undefined/*installed later*/, + /** + When using sqlite3_open_v2() it is important to keep the following + in mind: + + https://www.sqlite.org/c3ref/open.html + + - The flags for use with its 3rd argument are installed in this + object using the C-cide names, e.g. SQLITE_OPEN_CREATE. + + - If the combination of flags passed to it are invalid, + behavior is undefined. Thus is is never okay to call this + with fewer than 3 arguments, as JS will default the + missing arguments to `undefined`, which will result in a + flag value of 0. Most of the available SQLITE_OPEN_xxx + flags are meaningless in the WASM build, e.g. the mutext- + and cache-related flags, but they are retained in this + API for consistency's sake. + + - The final argument to this function specifies the VFS to + use, which is largely (but not entirely!) meaningless in + the WASM environment. It should always be null or + undefined, and it is safe to elide that argument when + calling this function. + */ + sqlite3_open_v2: function(filename,dbPtrPtr,flags,vfsStr){}/*installed later*/, + /** + The sqlite3_prepare_v3() binding handles two different uses + with differing JS/WASM semantics: + + 1) sqlite3_prepare_v3(pDb, sqlString, -1, prepFlags, ppStmt [, null]) + + 2) sqlite3_prepare_v3(pDb, sqlPointer, sqlByteLen, prepFlags, ppStmt, sqlPointerToPointer) + + Note that the SQL length argument (the 3rd argument) must, for + usage (1), always be negative because it must be a byte length + and that value is expensive to calculate from JS (where only + the character length of strings is readily available). It is + retained in this API's interface for code/documentation + compatibility reasons but is currently _always_ ignored. With + usage (2), the 3rd argument is used as-is but is is still + critical that the C-style input string (2nd argument) be + terminated with a 0 byte. + + In usage (1), the 2nd argument must be of type string, + Uint8Array, or Int8Array (either of which is assumed to + hold SQL). If it is, this function assumes case (1) and + calls the underyling C function with the equivalent of: + + (pDb, sqlAsString, -1, prepFlags, ppStmt, null) + + The pzTail argument is ignored in this case because its result + is meaningless when a string-type value is passed through + (because the string goes through another level of internal + conversion for WASM's sake and the result pointer would refer + to that transient conversion's memory, not the passed-in + string). + + If the sql argument is not a string, it must be a _pointer_ to + a NUL-terminated string which was allocated in the WASM memory + (e.g. using cwapi.wasm.alloc() or equivalent). In that case, + the final argument may be 0/null/undefined or must be a pointer + to which the "tail" of the compiled SQL is written, as + documented for the C-side sqlite3_prepare_v3(). In case (2), + the underlying C function is called with the equivalent of: + + (pDb, sqlAsPointer, (sqlByteLen||-1), prepFlags, ppStmt, pzTail) + + It returns its result and compiled statement as documented in + the C API. Fetching the output pointers (5th and 6th + parameters) requires using capi.wasm.getMemValue() (or + equivalent) and the pzTail will point to an address relative to + the sqlAsPointer value. + + If passed an invalid 2nd argument type, this function will + return SQLITE_MISUSE but will unfortunately be able to return + any additional error information because we have no way to set + the db's error state such that this function could return a + non-0 integer and the client could call sqlite3_errcode() or + sqlite3_errmsg() to fetch it. See the RFE at: + + https://sqlite.org/forum/forumpost/f9eb79b11aefd4fc81d + + The alternative would be to throw an exception for that case, + but that would be in strong constrast to the rest of the + C-level API and seems likely to cause more confusion. + + Side-note: in the C API the function does not fail if provided + an empty string but its result output pointer will be NULL. + */ + sqlite3_prepare_v3: function(dbPtr, sql, sqlByteLen, prepFlags, + stmtPtrPtr, strPtrPtr){}/*installed later*/, + + /** + Equivalent to calling sqlite3_prapare_v3() with 0 as its 4th argument. + */ + sqlite3_prepare_v2: function(dbPtr, sql, sqlByteLen, stmtPtrPtr, + strPtrPtr){}/*installed later*/, + + /** + Various internal-use utilities are added here as needed. They + are bound to an object only so that we have access to them in + the differently-scoped steps of the API bootstrapping + process. At the end of the API setup process, this object gets + removed. + */ + util:{ + isInt32, isTypedArray, isBindableTypedArray, isSQLableTypedArray, + affirmBindableTypedArray, typedArrayToString + }, + + /** + Holds state which are specific to the WASM-related + infrastructure and glue code. It is not expected that client + code will normally need these, but they're exposed here in case + it does. These APIs are _not_ to be considered an + official/stable part of the sqlite3 WASM API. They may change + as the developers' experience suggests appropriate changes. + + Note that a number of members of this object are injected + dynamically after the api object is fully constructed, so + not all are documented inline here. + */ + wasm: { + //^^^ TODO?: move wasm from sqlite3.capi.wasm to sqlite3.wasm + /** + Emscripten APIs have a deep-seated assumption that all pointers + are 32 bits. We'll remain optimistic that that won't always be + the case and will use this constant in places where we might + otherwise use a hard-coded 4. + */ + ptrSizeof: config.wasmPtrSizeof || 4, + /** + The WASM IR (Intermediate Representation) value for + pointer-type values. It MUST refer to a value type of the + size described by this.ptrSizeof _or_ it may be any value + which ends in '*', which Emscripten's glue code internally + translates to i32. + */ + ptrIR: config.wasmPtrIR || "i32", + /** + True if BigInt support was enabled via (e.g.) the + Emscripten -sWASM_BIGINT flag, else false. When + enabled, certain 64-bit sqlite3 APIs are enabled which + are not otherwise enabled due to JS/WASM int64 + impedence mismatches. + */ + bigIntEnabled: !!config.bigIntEnabled, + /** + The symbols exported by the WASM environment. + */ + exports: config.exports + || toss("Missing API config.exports (WASM module exports)."), + + /** + When Emscripten compiles with `-sIMPORT_MEMORY`, it + initalizes the heap and imports it into wasm, as opposed to + the other way around. In this case, the memory is not + available via this.exports.memory. + */ + memory: config.memory || config.exports['memory'] + || toss("API config object requires a WebAssembly.Memory object", + "in either config.exports.memory (exported)", + "or config.memory (imported)."), + /* Many more wasm-related APIs get installed later on. */ + }/*wasm*/ + }/*capi*/; + + /** + capi.wasm.alloc()'s srcTypedArray.byteLength bytes, + populates them with the values from the source + TypedArray, and returns the pointer to that memory. The + returned pointer must eventually be passed to + capi.wasm.dealloc() to clean it up. + + As a special case, to avoid further special cases where + this is used, if srcTypedArray.byteLength is 0, it + allocates a single byte and sets it to the value + 0. Even in such cases, calls must behave as if the + allocated memory has exactly srcTypedArray.byteLength + bytes. + + ACHTUNG: this currently only works for Uint8Array and + Int8Array types and will throw if srcTypedArray is of + any other type. + */ + capi.wasm.mallocFromTypedArray = function(srcTypedArray){ + affirmBindableTypedArray(srcTypedArray); + const pRet = this.alloc(srcTypedArray.byteLength || 1); + this.heapForSize(srcTypedArray.constructor).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); + return pRet; + }.bind(capi.wasm); + + const keyAlloc = config.allocExportName || 'malloc', + keyDealloc = config.deallocExportName || 'free'; + for(const key of [keyAlloc, keyDealloc]){ + const f = capi.wasm.exports[key]; + if(!(f instanceof Function)) toss("Missing required exports[",key,"] function."); + } + capi.wasm.alloc = function(n){ + const m = this.exports[keyAlloc](n); + if(!m) throw new WasmAllocError("Failed to allocate "+n+" bytes."); + return m; + }.bind(capi.wasm) + capi.wasm.dealloc = (m)=>capi.wasm.exports[keyDealloc](m); + + /** + Reports info about compile-time options using + sqlite_compileoption_get() and sqlite3_compileoption_used(). It + has several distinct uses: + + If optName is an array then it is expected to be a list of + compilation options and this function returns an object + which maps each such option to true or false, indicating + whether or not the given option was included in this + build. That object is returned. + + If optName is an object, its keys are expected to be compilation + options and this function sets each entry to true or false, + indicating whether the compilation option was used or not. That + object is returned. + + If passed no arguments then it returns an object mapping + all known compilation options to their compile-time values, + or boolean true if they are defined with no value. This + result, which is relatively expensive to compute, is cached + and returned for future no-argument calls. + + In all other cases it returns true if the given option was + active when when compiling the sqlite3 module, else false. + + Compile-time option names may optionally include their + "SQLITE_" prefix. When it returns an object of all options, + the prefix is elided. + */ + capi.wasm.compileOptionUsed = function f(optName){ + if(!arguments.length){ + if(f._result) return f._result; + else if(!f._opt){ + f._rx = /^([^=]+)=(.+)/; + f._rxInt = /^-?\d+$/; + f._opt = function(opt, rv){ + const m = f._rx.exec(opt); + rv[0] = (m ? m[1] : opt); + rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true; + }; + } + const rc = {}, ov = [0,0]; + let i = 0, k; + while((k = capi.sqlite3_compileoption_get(i++))){ + f._opt(k,ov); + rc[ov[0]] = ov[1]; + } + return f._result = rc; + }else if(Array.isArray(optName)){ + const rc = {}; + optName.forEach((v)=>{ + rc[v] = capi.sqlite3_compileoption_used(v); + }); + return rc; + }else if('object' === typeof optName){ + Object.keys(optName).forEach((k)=> { + optName[k] = capi.sqlite3_compileoption_used(k); + }); + return optName; + } + return ( + 'string'===typeof optName + ) ? !!capi.sqlite3_compileoption_used(optName) : false; + }/*compileOptionUsed()*/; + + capi.wasm.bindingSignatures = [ + /** + Signatures for the WASM-exported C-side functions. Each entry + is an array with 2+ elements: + + ["c-side name", + "result type" (capi.wasm.xWrap() syntax), + [arg types in xWrap() syntax] + // ^^^ this needn't strictly be an array: it can be subsequent + // elements instead: [x,y,z] is equivalent to x,y,z + ] + */ + // Please keep these sorted by function name! + ["sqlite3_bind_blob","int", "sqlite3_stmt*", "int", "*", "int", "*"], + ["sqlite3_bind_double","int", "sqlite3_stmt*", "int", "f64"], + ["sqlite3_bind_int","int", "sqlite3_stmt*", "int", "int"], + ["sqlite3_bind_null",undefined, "sqlite3_stmt*", "int"], + ["sqlite3_bind_parameter_count", "int", "sqlite3_stmt*"], + ["sqlite3_bind_parameter_index","int", "sqlite3_stmt*", "string"], + ["sqlite3_bind_text","int", "sqlite3_stmt*", "int", "string", "int", "int"], + ["sqlite3_close_v2", "int", "sqlite3*"], + ["sqlite3_changes", "int", "sqlite3*"], + ["sqlite3_clear_bindings","int", "sqlite3_stmt*"], + ["sqlite3_column_blob","*", "sqlite3_stmt*", "int"], + ["sqlite3_column_bytes","int", "sqlite3_stmt*", "int"], + ["sqlite3_column_count", "int", "sqlite3_stmt*"], + ["sqlite3_column_double","f64", "sqlite3_stmt*", "int"], + ["sqlite3_column_int","int", "sqlite3_stmt*", "int"], + ["sqlite3_column_name","string", "sqlite3_stmt*", "int"], + ["sqlite3_column_text","string", "sqlite3_stmt*", "int"], + ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], + ["sqlite3_compileoption_get", "string", "int"], + ["sqlite3_compileoption_used", "int", "string"], + ["sqlite3_create_function_v2", "int", + "sqlite3*", "string", "int", "int", "*", "*", "*", "*", "*"], + ["sqlite3_data_count", "int", "sqlite3_stmt*"], + ["sqlite3_db_filename", "string", "sqlite3*", "string"], + ["sqlite3_db_name", "string", "sqlite3*", "int"], + ["sqlite3_errmsg", "string", "sqlite3*"], + ["sqlite3_error_offset", "int", "sqlite3*"], + ["sqlite3_errstr", "string", "int"], + //["sqlite3_exec", "int", "sqlite3*", "string", "*", "*", "**"], + // ^^^ TODO: we need a wrapper to support passing a function pointer or a function + // for the callback. + ["sqlite3_expanded_sql", "string", "sqlite3_stmt*"], + ["sqlite3_extended_errcode", "int", "sqlite3*"], + ["sqlite3_extended_result_codes", "int", "sqlite3*", "int"], + ["sqlite3_finalize", "int", "sqlite3_stmt*"], + ["sqlite3_initialize", undefined], + ["sqlite3_interrupt", undefined, "sqlite3*" + /* ^^^ we cannot actually currently support this because JS is + single-threaded and we don't have a portable way to access a DB + from 2 SharedWorkers concurrently. */], + ["sqlite3_libversion", "string"], + ["sqlite3_libversion_number", "int"], + ["sqlite3_open", "int", "string", "*"], + ["sqlite3_open_v2", "int", "string", "*", "int", "string"], + /* sqlite3_prepare_v2() and sqlite3_prepare_v3() are handled + separately due to us requiring two different sets of semantics + for those, depending on how their SQL argument is provided. */ + ["sqlite3_reset", "int", "sqlite3_stmt*"], + ["sqlite3_result_blob",undefined, "*", "*", "int", "*"], + ["sqlite3_result_double",undefined, "*", "f64"], + ["sqlite3_result_error",undefined, "*", "string", "int"], + ["sqlite3_result_error_code", undefined, "*", "int"], + ["sqlite3_result_error_nomem", undefined, "*"], + ["sqlite3_result_error_toobig", undefined, "*"], + ["sqlite3_result_int",undefined, "*", "int"], + ["sqlite3_result_null",undefined, "*"], + ["sqlite3_result_text",undefined, "*", "string", "int", "*"], + ["sqlite3_sourceid", "string"], + ["sqlite3_sql", "string", "sqlite3_stmt*"], + ["sqlite3_step", "int", "sqlite3_stmt*"], + ["sqlite3_strglob", "int", "string","string"], + ["sqlite3_strlike", "int", "string","string","int"], + ["sqlite3_total_changes", "int", "sqlite3*"], + ["sqlite3_value_blob", "*", "*"], + ["sqlite3_value_bytes","int", "*"], + ["sqlite3_value_double","f64", "*"], + ["sqlite3_value_text", "string", "*"], + ["sqlite3_value_type", "int", "*"], + ["sqlite3_vfs_find", "*", "string"], + ["sqlite3_vfs_register", "int", "*", "int"] + ]/*capi.wasm.bindingSignatures*/; + + if(false && capi.wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ + /* ^^^ "the problem" is that this is an option feature and the + build-time function-export list does not currently take + optional features into account. */ + capi.wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); + } + + /** + Functions which require BigInt (int64) support are separated from + the others because we need to conditionally bind them or apply + dummy impls, depending on the capabilities of the environment. + */ + capi.wasm.bindingSignatures.int64 = [ + ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]], + ["sqlite3_changes64","i64", ["sqlite3*"]], + ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]], + ["sqlite3_total_changes64", "i64", ["sqlite3*"]] + ]; + + /* The remainder of the API will be set up in later steps. */ + return { + capi, + postInit: [ + /* some pieces of the API may install functions into this array, + and each such function will be called, passed (self,sqlite3), + at the very end of the API load/init process, where self is + the current global object and sqlite3 is the object returned + from sqlite3ApiBootstrap(). This array will be removed at the + end of the API setup process. */], + /** Config is needed downstream for gluing pieces together. It + will be removed at the end of the API setup process. */ + config + }; +}/*sqlite3ApiBootstrap()*/; diff --git a/ext/wasm/api/sqlite3-api-worker.js b/ext/wasm/api/sqlite3-api-worker.js new file mode 100644 index 0000000000..1d13d4ed6b --- /dev/null +++ b/ext/wasm/api/sqlite3-api-worker.js @@ -0,0 +1,421 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements a Worker-based wrapper around SQLite3 OO API + #1. + + In order to permit this API to be loaded in worker threads without + automatically registering onmessage handlers, initializing the + worker API requires calling initWorkerAPI(). If this function + is called from a non-worker thread then it throws an exception. + + When initialized, it installs message listeners to receive messages + from the main thread and then it posts a message in the form: + + ``` + {type:'sqlite3-api',data:'worker-ready'} + ``` + + This file requires that the core C-style sqlite3 API and OO API #1 + have been loaded and that self.sqlite3 contains both, + as documented for those APIs. +*/ +self.sqlite3.initWorkerAPI = function(){ + 'use strict'; + /** + UNDER CONSTRUCTION + + We need an API which can proxy the DB API via a Worker message + interface. The primary quirky factor in such an API is that we + cannot pass callback functions between the window thread and a + worker thread, so we have to receive all db results via + asynchronous message-passing. That requires an asychronous API + with a distinctly different shape that the main OO API. + + Certain important considerations here include: + + - Support only one db connection or multiple? The former is far + easier, but there's always going to be a user out there who wants + to juggle six database handles at once. Do we add that complexity + or tell such users to write their own code using the provided + lower-level APIs? + + - Fetching multiple results: do we pass them on as a series of + messages, with start/end messages on either end, or do we collect + all results and bundle them back in a single message? The former + is, generically speaking, more memory-efficient but the latter + far easier to implement in this environment. The latter is + untennable for large data sets. Despite a web page hypothetically + being a relatively limited environment, there will always be + those users who feel that they should/need to be able to work + with multi-hundred-meg (or larger) blobs, and passing around + arrays of those may quickly exhaust the JS engine's memory. + + TODOs include, but are not limited to: + + - The ability to manage multiple DB handles. This can + potentially be done via a simple mapping of DB.filename or + DB.pointer (`sqlite3*` handle) to DB objects. The open() + interface would need to provide an ID (probably DB.pointer) back + to the user which can optionally be passed as an argument to + the other APIs (they'd default to the first-opened DB, for + ease of use). Client-side usability of this feature would + benefit from making another wrapper class (or a singleton) + available to the main thread, with that object proxying all(?) + communication with the worker. + + - Revisit how virtual files are managed. We currently delete DBs + from the virtual filesystem when we close them, for the sake of + saving memory (the VFS lives in RAM). Supporting multiple DBs may + require that we give up that habit. Similarly, fully supporting + ATTACH, where a user can upload multiple DBs and ATTACH them, + also requires the that we manage the VFS entries better. + */ + const toss = (...args)=>{throw new Error(args.join(' '))}; + if('function' !== typeof importScripts){ + toss("Cannot initalize the sqlite3 worker API in the main thread."); + } + /* This is a web worker, so init the worker-based API. */ + const self = this.self; + const sqlite3 = this.sqlite3 || toss("Missing self.sqlite3 object."); + const SQLite3 = sqlite3.oo1 || toss("Missing self.sqlite3.oo1 OO API."); + const DB = SQLite3.DB; + + /** + Returns the app-wide unique ID for the given db, creating one if + needed. + */ + const getDbId = function(db){ + let id = wState.idMap.get(db); + if(id) return id; + id = 'db#'+(++wState.idSeq)+'@'+db.pointer; + /** ^^^ can't simply use db.pointer b/c closing/opening may re-use + the same address, which could map pending messages to a wrong + instance. */ + wState.idMap.set(db, id); + return id; + }; + + /** + Helper for managing Worker-level state. + */ + const wState = { + defaultDb: undefined, + idSeq: 0, + idMap: new WeakMap, + open: function(arg){ + // TODO: if arg is a filename, look for a db in this.dbs with the + // same filename and close/reopen it (or just pass it back as is?). + if(!arg && this.defaultDb) return this.defaultDb; + //???if(this.defaultDb) this.defaultDb.close(); + let db; + db = (Array.isArray(arg) ? new DB(...arg) : new DB(arg)); + this.dbs[getDbId(db)] = db; + if(!this.defaultDb) this.defaultDb = db; + return db; + }, + close: function(db,alsoUnlink){ + if(db){ + delete this.dbs[getDbId(db)]; + db.close(alsoUnlink); + if(db===this.defaultDb) this.defaultDb = undefined; + } + }, + post: function(type,data,xferList){ + if(xferList){ + self.postMessage({type, data},xferList); + xferList.length = 0; + }else{ + self.postMessage({type, data}); + } + }, + /** Map of DB IDs to DBs. */ + dbs: Object.create(null), + getDb: function(id,require=true){ + return this.dbs[id] + || (require ? toss("Unknown (or closed) DB ID:",id) : undefined); + } + }; + + /** Throws if the given db is falsy or not opened. */ + const affirmDbOpen = function(db = wState.defaultDb){ + return (db && db.pointer) ? db : toss("DB is not opened."); + }; + + /** Extract dbId from the given message payload. */ + const getMsgDb = function(msgData,affirmExists=true){ + const db = wState.getDb(msgData.dbId,false) || wState.defaultDb; + return affirmExists ? affirmDbOpen(db) : db; + }; + + const getDefaultDbId = function(){ + return wState.defaultDb && getDbId(wState.defaultDb); + }; + + /** + A level of "organizational abstraction" for the Worker + API. Each method in this object must map directly to a Worker + message type key. The onmessage() dispatcher attempts to + dispatch all inbound messages to a method of this object, + passing it the event.data part of the inbound event object. All + methods must return a plain Object containing any response + state, which the dispatcher may amend. All methods must throw + on error. + */ + const wMsgHandler = { + xfer: [/*Temp holder for "transferable" postMessage() state.*/], + /** + Proxy for DB.exec() which expects a single argument of type + string (SQL to execute) or an options object in the form + expected by exec(). The notable differences from exec() + include: + + - The default value for options.rowMode is 'array' because + the normal default cannot cross the window/Worker boundary. + + - A function-type options.callback property cannot cross + the window/Worker boundary, so is not useful here. If + options.callback is a string then it is assumed to be a + message type key, in which case a callback function will be + applied which posts each row result via: + + postMessage({type: thatKeyType, data: theRow}) + + And, at the end of the result set (whether or not any + result rows were produced), it will post an identical + message with data:null to alert the caller than the result + set is completed. + + The callback proxy must not recurse into this interface, or + results are undefined. (It hypothetically cannot recurse + because an exec() call will be tying up the Worker thread, + causing any recursion attempt to wait until the first + exec() is completed.) + + The response is the input options object (or a synthesized + one if passed only a string), noting that + options.resultRows and options.columnNames may be populated + by the call to exec(). + + This opens/creates the Worker's db if needed. + */ + exec: function(ev){ + const opt = ( + 'string'===typeof ev.data + ) ? {sql: ev.data} : (ev.data || Object.create(null)); + if(undefined===opt.rowMode){ + /* Since the default rowMode of 'stmt' is not useful + for the Worker interface, we'll default to + something else. */ + opt.rowMode = 'array'; + }else if('stmt'===opt.rowMode){ + toss("Invalid rowMode for exec(): stmt mode", + "does not work in the Worker API."); + } + const db = getMsgDb(ev); + if(opt.callback || Array.isArray(opt.resultRows)){ + // Part of a copy-avoidance optimization for blobs + db._blobXfer = this.xfer; + } + const callbackMsgType = opt.callback; + if('string' === typeof callbackMsgType){ + /* Treat this as a worker message type and post each + row as a message of that type. */ + const that = this; + opt.callback = + (row)=>wState.post(callbackMsgType,row,this.xfer); + } + try { + db.exec(opt); + if(opt.callback instanceof Function){ + opt.callback = callbackMsgType; + wState.post(callbackMsgType, null); + } + }/*catch(e){ + console.warn("Worker is propagating:",e);throw e; + }*/finally{ + delete db._blobXfer; + if(opt.callback){ + opt.callback = callbackMsgType; + } + } + return opt; + }/*exec()*/, + /** + TO(re)DO, once we can abstract away access to the + JS environment's virtual filesystem. Currently this + always throws. + + Response is (should be) an object: + + { + buffer: Uint8Array (db file contents), + filename: the current db filename, + mimetype: 'application/x-sqlite3' + } + + TODO is to determine how/whether this feature can support + exports of ":memory:" and "" (temp file) DBs. The latter is + ostensibly easy because the file is (potentially) on disk, but + the former does not have a structure which maps directly to a + db file image. + */ + export: function(ev){ + toss("export() requires reimplementing for portability reasons."); + /**const db = getMsgDb(ev); + const response = { + buffer: db.exportBinaryImage(), + filename: db.filename, + mimetype: 'application/x-sqlite3' + }; + this.xfer.push(response.buffer.buffer); + return response;**/ + }/*export()*/, + /** + Proxy for the DB constructor. Expects to be passed a single + object or a falsy value to use defaults. The object may + have a filename property to name the db file (see the DB + constructor for peculiarities and transformations) and/or a + buffer property (a Uint8Array holding a complete database + file's contents). The response is an object: + + { + filename: db filename (possibly differing from the input), + + id: an opaque ID value intended for future distinction + between multiple db handles. Messages including a specific + ID will use the DB for that ID. + + } + + If the Worker's db is currently opened, this call closes it + before proceeding. + */ + open: function(ev){ + wState.close(/*true???*/); + const args = [], data = (ev.data || {}); + if(data.simulateError){ + toss("Throwing because of open.simulateError flag."); + } + if(data.filename) args.push(data.filename); + if(data.buffer){ + args.push(data.buffer); + this.xfer.push(data.buffer.buffer); + } + const db = wState.open(args); + return { + filename: db.filename, + dbId: getDbId(db) + }; + }, + /** + Proxy for DB.close(). If ev.data may either be a boolean or + an object with an `unlink` property. If that value is + truthy then the db file (if the db is currently open) will + be unlinked from the virtual filesystem, else it will be + kept intact. The response object is: + + { + filename: db filename _if_ the db is opened when this + is called, else the undefined value + } + */ + close: function(ev){ + const db = getMsgDb(ev,false); + const response = { + filename: db && db.filename + }; + if(db){ + wState.close(db, !!((ev.data && 'object'===typeof ev.data) + ? ev.data.unlink : ev.data)); + } + return response; + }, + toss: function(ev){ + toss("Testing worker exception"); + } + }/*wMsgHandler*/; + + /** + UNDER CONSTRUCTION! + + A subset of the DB API is accessible via Worker messages in the + form: + + { type: apiCommand, + dbId: optional DB ID value (not currently used!) + data: apiArguments + } + + As a rule, these commands respond with a postMessage() of their + own in the same form, but will, if needed, transform the `data` + member to an object and may add state to it. The responses + always have an object-format `data` part. If the inbound `data` + is an object which has a `messageId` property, that property is + always mirrored in the result object, for use in client-side + dispatching of these asynchronous results. Exceptions thrown + during processing result in an `error`-type event with a + payload in the form: + + { + message: error string, + errorClass: class name of the error type, + dbId: DB handle ID, + input: ev.data, + [messageId: if set in the inbound message] + } + + The individual APIs are documented in the wMsgHandler object. + */ + self.onmessage = function(ev){ + ev = ev.data; + let response, dbId = ev.dbId, evType = ev.type; + const arrivalTime = performance.now(); + try { + if(wMsgHandler.hasOwnProperty(evType) && + wMsgHandler[evType] instanceof Function){ + response = wMsgHandler[evType](ev); + }else{ + toss("Unknown db worker message type:",ev.type); + } + }catch(err){ + evType = 'error'; + response = { + message: err.message, + errorClass: err.name, + input: ev + }; + if(err.stack){ + response.stack = ('string'===typeof err.stack) + ? err.stack.split('\n') : err.stack; + } + if(0) console.warn("Worker is propagating an exception to main thread.", + "Reporting it _here_ for the stack trace:",err,response); + } + if(!response.messageId && ev.data + && 'object'===typeof ev.data && ev.data.messageId){ + response.messageId = ev.data.messageId; + } + if(!dbId){ + dbId = response.dbId/*from 'open' cmd*/ + || getDefaultDbId(); + } + if(!response.dbId) response.dbId = dbId; + // Timing info is primarily for use in testing this API. It's not part of + // the public API. arrivalTime = when the worker got the message. + response.workerReceivedTime = arrivalTime; + response.workerRespondTime = performance.now(); + response.departureTime = ev.departureTime; + wState.post(evType, response, wMsgHandler.xfer); + }; + setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'worker-ready'}), 0); +}.bind({self, sqlite3: self.sqlite3}); diff --git a/ext/wasm/api/sqlite3-wasi.h b/ext/wasm/api/sqlite3-wasi.h new file mode 100644 index 0000000000..096f45dfec --- /dev/null +++ b/ext/wasm/api/sqlite3-wasi.h @@ -0,0 +1,69 @@ +/** + Dummy function stubs to get sqlite3.c compiling with + wasi-sdk. This requires, in addition: + + -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_GETPID + + -lwasi-emulated-getpid +*/ +typedef unsigned mode_t; +int fchmod(int fd, mode_t mode); +int fchmod(int fd, mode_t mode){ + return (fd && mode) ? 0 : 0; +} +typedef unsigned uid_t; +typedef uid_t gid_t; +int fchown(int fd, uid_t owner, gid_t group); +int fchown(int fd, uid_t owner, gid_t group){ + return (fd && owner && group) ? 0 : 0; +} +uid_t geteuid(void); +uid_t geteuid(void){return 0;} +#if !defined(F_WRLCK) +enum { +F_WRLCK, +F_RDLCK, +F_GETLK, +F_SETLK, +F_UNLCK +}; +#endif + +#undef HAVE_PREAD + +#include +#define WASM__KEEP __attribute__((used)) + +#if 0 +/** + wasi-sdk cannot build sqlite3's default VFS without at least the following + functions. They are apparently syscalls which clients have to implement or + otherwise obtain. + + https://github.com/WebAssembly/WASI/blob/main/phases/snapshot/docs.md +*/ +environ_get +environ_sizes_get +clock_time_get +fd_close +fd_fdstat_get +fd_fdstat_set_flags +fd_filestat_get +fd_filestat_set_size +fd_pread +fd_prestat_get +fd_prestat_dir_name +fd_read +fd_seek +fd_sync +fd_write +path_create_directory +path_filestat_get +path_filestat_set_times +path_open +path_readlink +path_remove_directory +path_unlink_file +poll_oneoff +proc_exit +#endif diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c new file mode 100644 index 0000000000..c27dad56ff --- /dev/null +++ b/ext/wasm/api/sqlite3-wasm.c @@ -0,0 +1,413 @@ +#include "sqlite3.c" + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** For purposes of certain hand-crafted C/Wasm function bindings, we +** need a way of reporting errors which is consistent with the rest of +** the C API. To that end, this internal-use-only function is a thin +** proxy around sqlite3ErrorWithMessage(). The intent is that it only +** be used from Wasm bindings such as sqlite3_prepare_v2/v3(), and +** definitely not from client code. +** +** Returns err_code. +*/ +int sqlite3_wasm_db_error(sqlite3*db, int err_code, + const char *zMsg){ + if(0!=zMsg){ + const int nMsg = sqlite3Strlen30(zMsg); + sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); + }else{ + sqlite3ErrorWithMsg(db, err_code, NULL); + } + return err_code; +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. Unlike the +** rest of the sqlite3 API, this part requires C99 for snprintf() and +** variadic macros. +** +** Returns a string containing a JSON-format "enum" of C-level +** constants intended to be imported into the JS environment. The JSON +** is initialized the first time this function is called and that +** result is reused for all future calls. +** +** If this function returns NULL then it means that the internal +** buffer is not large enough for the generated JSON. In debug builds +** that will trigger an assert(). +*/ +const char * sqlite3_wasm_enum_json(void){ + static char strBuf[1024 * 8] = {0} /* where the JSON goes */; + int n = 0, childCount = 0, structCount = 0 + /* output counters for figuring out where commas go */; + char * pos = &strBuf[1] /* skip first byte for now to help protect + ** against a small race condition */; + char const * const zEnd = pos + sizeof(strBuf) /* one-past-the-end */; + if(strBuf[0]) return strBuf; + /* Leave strBuf[0] at 0 until the end to help guard against a tiny + ** race condition. If this is called twice concurrently, they might + ** end up both writing to strBuf, but they'll both write the same + ** thing, so that's okay. If we set byte 0 up front then the 2nd + ** instance might return and use the string before the 1st instance + ** is done filling it. */ + +/* Core output macros... */ +#define lenCheck assert(pos < zEnd - 128 \ + && "sqlite3_wasm_enum_json() buffer is too small."); \ + if(pos >= zEnd - 128) return 0 +#define outf(format,...) \ + pos += snprintf(pos, ((size_t)(zEnd - pos)), format, __VA_ARGS__); \ + lenCheck +#define out(TXT) outf("%s",TXT) +#define CloseBrace(LEVEL) \ + assert(LEVEL<5); memset(pos, '}', LEVEL); pos+=LEVEL; lenCheck + +/* Macros for emitting maps of integer- and string-type macros to +** their values. */ +#define DefGroup(KEY) n = 0; \ + outf("%s\"" #KEY "\": {",(childCount++ ? "," : "")); +#define DefInt(KEY) \ + outf("%s\"%s\": %d", (n++ ? ", " : ""), #KEY, (int)KEY) +#define DefStr(KEY) \ + outf("%s\"%s\": \"%s\"", (n++ ? ", " : ""), #KEY, KEY) +#define _DefGroup CloseBrace(1) + + DefGroup(version) { + DefInt(SQLITE_VERSION_NUMBER); + DefStr(SQLITE_VERSION); + DefStr(SQLITE_SOURCE_ID); + } _DefGroup; + + DefGroup(resultCodes) { + DefInt(SQLITE_OK); + DefInt(SQLITE_ERROR); + DefInt(SQLITE_INTERNAL); + DefInt(SQLITE_PERM); + DefInt(SQLITE_ABORT); + DefInt(SQLITE_BUSY); + DefInt(SQLITE_LOCKED); + DefInt(SQLITE_NOMEM); + DefInt(SQLITE_READONLY); + DefInt(SQLITE_INTERRUPT); + DefInt(SQLITE_IOERR); + DefInt(SQLITE_CORRUPT); + DefInt(SQLITE_NOTFOUND); + DefInt(SQLITE_FULL); + DefInt(SQLITE_CANTOPEN); + DefInt(SQLITE_PROTOCOL); + DefInt(SQLITE_EMPTY); + DefInt(SQLITE_SCHEMA); + DefInt(SQLITE_TOOBIG); + DefInt(SQLITE_CONSTRAINT); + DefInt(SQLITE_MISMATCH); + DefInt(SQLITE_MISUSE); + DefInt(SQLITE_NOLFS); + DefInt(SQLITE_AUTH); + DefInt(SQLITE_FORMAT); + DefInt(SQLITE_RANGE); + DefInt(SQLITE_NOTADB); + DefInt(SQLITE_NOTICE); + DefInt(SQLITE_WARNING); + DefInt(SQLITE_ROW); + DefInt(SQLITE_DONE); + + // Extended Result Codes + DefInt(SQLITE_ERROR_MISSING_COLLSEQ); + DefInt(SQLITE_ERROR_RETRY); + DefInt(SQLITE_ERROR_SNAPSHOT); + DefInt(SQLITE_IOERR_READ); + DefInt(SQLITE_IOERR_SHORT_READ); + DefInt(SQLITE_IOERR_WRITE); + DefInt(SQLITE_IOERR_FSYNC); + DefInt(SQLITE_IOERR_DIR_FSYNC); + DefInt(SQLITE_IOERR_TRUNCATE); + DefInt(SQLITE_IOERR_FSTAT); + DefInt(SQLITE_IOERR_UNLOCK); + DefInt(SQLITE_IOERR_RDLOCK); + DefInt(SQLITE_IOERR_DELETE); + DefInt(SQLITE_IOERR_BLOCKED); + DefInt(SQLITE_IOERR_NOMEM); + DefInt(SQLITE_IOERR_ACCESS); + DefInt(SQLITE_IOERR_CHECKRESERVEDLOCK); + DefInt(SQLITE_IOERR_LOCK); + DefInt(SQLITE_IOERR_CLOSE); + DefInt(SQLITE_IOERR_DIR_CLOSE); + DefInt(SQLITE_IOERR_SHMOPEN); + DefInt(SQLITE_IOERR_SHMSIZE); + DefInt(SQLITE_IOERR_SHMLOCK); + DefInt(SQLITE_IOERR_SHMMAP); + DefInt(SQLITE_IOERR_SEEK); + DefInt(SQLITE_IOERR_DELETE_NOENT); + DefInt(SQLITE_IOERR_MMAP); + DefInt(SQLITE_IOERR_GETTEMPPATH); + DefInt(SQLITE_IOERR_CONVPATH); + DefInt(SQLITE_IOERR_VNODE); + DefInt(SQLITE_IOERR_AUTH); + DefInt(SQLITE_IOERR_BEGIN_ATOMIC); + DefInt(SQLITE_IOERR_COMMIT_ATOMIC); + DefInt(SQLITE_IOERR_ROLLBACK_ATOMIC); + DefInt(SQLITE_IOERR_DATA); + DefInt(SQLITE_IOERR_CORRUPTFS); + DefInt(SQLITE_LOCKED_SHAREDCACHE); + DefInt(SQLITE_LOCKED_VTAB); + DefInt(SQLITE_BUSY_RECOVERY); + DefInt(SQLITE_BUSY_SNAPSHOT); + DefInt(SQLITE_BUSY_TIMEOUT); + DefInt(SQLITE_CANTOPEN_NOTEMPDIR); + DefInt(SQLITE_CANTOPEN_ISDIR); + DefInt(SQLITE_CANTOPEN_FULLPATH); + DefInt(SQLITE_CANTOPEN_CONVPATH); + //DefInt(SQLITE_CANTOPEN_DIRTYWAL)/*docs say not used*/; + DefInt(SQLITE_CANTOPEN_SYMLINK); + DefInt(SQLITE_CORRUPT_VTAB); + DefInt(SQLITE_CORRUPT_SEQUENCE); + DefInt(SQLITE_CORRUPT_INDEX); + DefInt(SQLITE_READONLY_RECOVERY); + DefInt(SQLITE_READONLY_CANTLOCK); + DefInt(SQLITE_READONLY_ROLLBACK); + DefInt(SQLITE_READONLY_DBMOVED); + DefInt(SQLITE_READONLY_CANTINIT); + DefInt(SQLITE_READONLY_DIRECTORY); + DefInt(SQLITE_ABORT_ROLLBACK); + DefInt(SQLITE_CONSTRAINT_CHECK); + DefInt(SQLITE_CONSTRAINT_COMMITHOOK); + DefInt(SQLITE_CONSTRAINT_FOREIGNKEY); + DefInt(SQLITE_CONSTRAINT_FUNCTION); + DefInt(SQLITE_CONSTRAINT_NOTNULL); + DefInt(SQLITE_CONSTRAINT_PRIMARYKEY); + DefInt(SQLITE_CONSTRAINT_TRIGGER); + DefInt(SQLITE_CONSTRAINT_UNIQUE); + DefInt(SQLITE_CONSTRAINT_VTAB); + DefInt(SQLITE_CONSTRAINT_ROWID); + DefInt(SQLITE_CONSTRAINT_PINNED); + DefInt(SQLITE_CONSTRAINT_DATATYPE); + DefInt(SQLITE_NOTICE_RECOVER_WAL); + DefInt(SQLITE_NOTICE_RECOVER_ROLLBACK); + DefInt(SQLITE_WARNING_AUTOINDEX); + DefInt(SQLITE_AUTH_USER); + DefInt(SQLITE_OK_LOAD_PERMANENTLY); + //DefInt(SQLITE_OK_SYMLINK) /* internal use only */; + } _DefGroup; + + DefGroup(dataTypes) { + DefInt(SQLITE_INTEGER); + DefInt(SQLITE_FLOAT); + DefInt(SQLITE_TEXT); + DefInt(SQLITE_BLOB); + DefInt(SQLITE_NULL); + } _DefGroup; + + DefGroup(encodings) { + /* Noting that the wasm binding only aims to support UTF-8. */ + DefInt(SQLITE_UTF8); + DefInt(SQLITE_UTF16LE); + DefInt(SQLITE_UTF16BE); + DefInt(SQLITE_UTF16); + /*deprecated DefInt(SQLITE_ANY); */ + DefInt(SQLITE_UTF16_ALIGNED); + } _DefGroup; + + DefGroup(blobFinalizers) { + /* SQLITE_STATIC/TRANSIENT need to be handled explicitly as + ** integers to avoid casting-related warnings. */ + out("\"SQLITE_STATIC\":0, " + "\"SQLITE_TRANSIENT\":-1"); + } _DefGroup; + + DefGroup(udfFlags) { + DefInt(SQLITE_DETERMINISTIC); + DefInt(SQLITE_DIRECTONLY); + DefInt(SQLITE_INNOCUOUS); + } _DefGroup; + + DefGroup(openFlags) { + /* Noting that not all of these will have any effect in WASM-space. */ + DefInt(SQLITE_OPEN_READONLY); + DefInt(SQLITE_OPEN_READWRITE); + DefInt(SQLITE_OPEN_CREATE); + DefInt(SQLITE_OPEN_URI); + DefInt(SQLITE_OPEN_MEMORY); + DefInt(SQLITE_OPEN_NOMUTEX); + DefInt(SQLITE_OPEN_FULLMUTEX); + DefInt(SQLITE_OPEN_SHAREDCACHE); + DefInt(SQLITE_OPEN_PRIVATECACHE); + DefInt(SQLITE_OPEN_EXRESCODE); + DefInt(SQLITE_OPEN_NOFOLLOW); + /* OPEN flags for use with VFSes... */ + DefInt(SQLITE_OPEN_MAIN_DB); + DefInt(SQLITE_OPEN_MAIN_JOURNAL); + DefInt(SQLITE_OPEN_TEMP_DB); + DefInt(SQLITE_OPEN_TEMP_JOURNAL); + DefInt(SQLITE_OPEN_TRANSIENT_DB); + DefInt(SQLITE_OPEN_SUBJOURNAL); + DefInt(SQLITE_OPEN_SUPER_JOURNAL); + DefInt(SQLITE_OPEN_WAL); + DefInt(SQLITE_OPEN_DELETEONCLOSE); + DefInt(SQLITE_OPEN_EXCLUSIVE); + } _DefGroup; + + DefGroup(syncFlags) { + DefInt(SQLITE_SYNC_NORMAL); + DefInt(SQLITE_SYNC_FULL); + DefInt(SQLITE_SYNC_DATAONLY); + } _DefGroup; + + DefGroup(prepareFlags) { + DefInt(SQLITE_PREPARE_PERSISTENT); + DefInt(SQLITE_PREPARE_NORMALIZE); + DefInt(SQLITE_PREPARE_NO_VTAB); + } _DefGroup; + + DefGroup(flock) { + DefInt(SQLITE_LOCK_NONE); + DefInt(SQLITE_LOCK_SHARED); + DefInt(SQLITE_LOCK_RESERVED); + DefInt(SQLITE_LOCK_PENDING); + DefInt(SQLITE_LOCK_EXCLUSIVE); + } _DefGroup; + + DefGroup(ioCap) { + DefInt(SQLITE_IOCAP_ATOMIC); + DefInt(SQLITE_IOCAP_ATOMIC512); + DefInt(SQLITE_IOCAP_ATOMIC1K); + DefInt(SQLITE_IOCAP_ATOMIC2K); + DefInt(SQLITE_IOCAP_ATOMIC4K); + DefInt(SQLITE_IOCAP_ATOMIC8K); + DefInt(SQLITE_IOCAP_ATOMIC16K); + DefInt(SQLITE_IOCAP_ATOMIC32K); + DefInt(SQLITE_IOCAP_ATOMIC64K); + DefInt(SQLITE_IOCAP_SAFE_APPEND); + DefInt(SQLITE_IOCAP_SEQUENTIAL); + DefInt(SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN); + DefInt(SQLITE_IOCAP_POWERSAFE_OVERWRITE); + DefInt(SQLITE_IOCAP_IMMUTABLE); + DefInt(SQLITE_IOCAP_BATCH_ATOMIC); + } _DefGroup; + + DefGroup(access){ + DefInt(SQLITE_ACCESS_EXISTS); + DefInt(SQLITE_ACCESS_READWRITE); + DefInt(SQLITE_ACCESS_READ)/*docs say this is unused*/; + } _DefGroup; + +#undef DefGroup +#undef DefStr +#undef DefInt +#undef _DefGroup + + /* + ** Emit an array of "StructBinder" struct descripions, which look + ** like: + ** + ** { + ** "name": "MyStruct", + ** "sizeof": 16, + ** "members": { + ** "member1": {"offset": 0,"sizeof": 4,"signature": "i"}, + ** "member2": {"offset": 4,"sizeof": 4,"signature": "p"}, + ** "member3": {"offset": 8,"sizeof": 8,"signature": "j"} + ** } + ** } + ** + ** Detailed documentation for those bits are in an external + ** file (StackBinder.md, as of this writing). + */ + + /** Macros for emitting StructBinder description. */ +#define StructBinder__(TYPE) \ + n = 0; \ + outf("%s{", (structCount++ ? ", " : "")); \ + out("\"name\": \"" # TYPE "\","); \ + outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ + out(",\"members\": {"); +#define StructBinder_(T) StructBinder__(T) + /** ^^^ indirection needed to expand CurrentStruct */ +#define StructBinder StructBinder_(CurrentStruct) +#define _StructBinder CloseBrace(2) +#define M(MEMBER,SIG) \ + outf("%s\"%s\": " \ + "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ + (n++ ? ", " : ""), #MEMBER, \ + (int)offsetof(CurrentStruct,MEMBER), \ + (int)sizeof(((CurrentStruct*)0)->MEMBER), \ + SIG) + + structCount = 0; + out(", \"structs\": ["); { + +#define CurrentStruct sqlite3_vfs + StructBinder { + M(iVersion,"i"); + M(szOsFile,"i"); + M(mxPathname,"i"); + M(pNext,"p"); + M(zName,"s"); + M(pAppData,"p"); + M(xOpen,"i(pppip)"); + M(xDelete,"i(ppi)"); + M(xAccess,"i(ppip)"); + M(xFullPathname,"i(ppip)"); + M(xDlOpen,"p(pp)"); + M(xDlError,"p(pip)"); + M(xDlSym,"p()"); + M(xDlClose,"v(pp)"); + M(xRandomness,"i(pip)"); + M(xSleep,"i(pi)"); + M(xCurrentTime,"i(pp)"); + M(xGetLastError,"i(pip)"); + M(xCurrentTimeInt64,"i(pp)"); + M(xSetSystemCall,"i(ppp)"); + M(xGetSystemCall,"p(pp)"); + M(xNextSystemCall,"p(pp)"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_io_methods + StructBinder { + M(iVersion,"i"); + M(xClose,"i(p)"); + M(xRead,"i(ppij)"); + M(xWrite,"i(ppij)"); + M(xTruncate,"i(pj)"); + M(xSync,"i(pi)"); + M(xFileSize,"i(pp)"); + M(xLock,"i(pi)"); + M(xUnlock,"i(pi)"); + M(xCheckReservedLock,"i(pp)"); + M(xFileControl,"i(pip)"); + M(xSectorSize,"i(p)"); + M(xDeviceCharacteristics,"i(p)"); + M(xShmMap,"i(piiip)"); + M(xShmLock,"i(piii)"); + M(xShmBarrier,"v(p)"); + M(xShmUnmap,"i(pi)"); + M(xFetch,"i(pjip)"); + M(xUnfetch,"i(pjp)"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_file + StructBinder { + M(pMethods,"P"); + } _StructBinder; +#undef CurrentStruct + + } out( "]"/*structs*/); + + out("}"/*top-level object*/); + *pos = 0; + strBuf[0] = '{'/*end of the race-condition workaround*/; + return strBuf; +#undef StructBinder +#undef StructBinder_ +#undef StructBinder__ +#undef M +#undef _StructBinder +#undef CloseBrace +#undef out +#undef outf +#undef lenCheck +} diff --git a/ext/wasm/api/sqlite3-worker.js b/ext/wasm/api/sqlite3-worker.js new file mode 100644 index 0000000000..48797de8ab --- /dev/null +++ b/ext/wasm/api/sqlite3-worker.js @@ -0,0 +1,31 @@ +/* + 2022-05-23 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This is a JS Worker file for the main sqlite3 api. It loads + sqlite3.js, initializes the module, and postMessage()'s a message + after the module is initialized: + + {type: 'sqlite3-api', data: 'worker-ready'} + + This seemingly superfluous level of indirection is necessary when + loading sqlite3.js via a Worker. Instantiating a worker with new + Worker("sqlite.js") will not (cannot) call sqlite3InitModule() to + initialize the module due to a timing/order-of-operations conflict + (and that symbol is not exported in a way that a Worker loading it + that way can see it). Thus JS code wanting to load the sqlite3 + Worker-specific API needs to pass _this_ file (or equivalent) to the + Worker constructor and then listen for an event in the form shown + above in order to know when the module has completed initialization. +*/ +"use strict"; +importScripts('sqlite3.js'); +sqlite3InitModule().then((EmscriptenModule)=>EmscriptenModule.sqlite3.initWorkerAPI()); diff --git a/ext/wasm/common/SqliteTestUtil.js b/ext/wasm/common/SqliteTestUtil.js new file mode 100644 index 0000000000..c7c99240e6 --- /dev/null +++ b/ext/wasm/common/SqliteTestUtil.js @@ -0,0 +1,173 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file contains bootstrapping code used by various test scripts + which live in this file's directory. +*/ +'use strict'; +(function(self){ + /* querySelectorAll() proxy */ + const EAll = function(/*[element=document,] cssSelector*/){ + return (arguments.length>1 ? arguments[0] : document) + .querySelectorAll(arguments[arguments.length-1]); + }; + /* querySelector() proxy */ + const E = function(/*[element=document,] cssSelector*/){ + return (arguments.length>1 ? arguments[0] : document) + .querySelector(arguments[arguments.length-1]); + }; + + /** + Helpers for writing sqlite3-specific tests. + */ + self.SqliteTestUtil = { + /** Running total of the number of tests run via + this API. */ + counter: 0, + /** + If expr is a function, it is called and its result + is returned, coerced to a bool, else expr, coerced to + a bool, is returned. + */ + toBool: function(expr){ + return (expr instanceof Function) ? !!expr() : !!expr; + }, + /** abort() if expr is false. If expr is a function, it + is called and its result is evaluated. + */ + assert: function f(expr, msg){ + if(!f._){ + f._ = ('undefined'===typeof abort + ? (msg)=>{throw new Error(msg)} + : abort); + } + ++this.counter; + if(!this.toBool(expr)){ + f._(msg || "Assertion failed."); + } + return this; + }, + /** Identical to assert() but throws instead of calling + abort(). */ + affirm: function(expr, msg){ + ++this.counter; + if(!this.toBool(expr)) throw new Error(msg || "Affirmation failed."); + return this; + }, + /** Calls f() and squelches any exception it throws. If it + does not throw, this function throws. */ + mustThrow: function(f, msg){ + ++this.counter; + let err; + try{ f(); } catch(e){err=e;} + if(!err) throw new Error(msg || "Expected exception."); + return this; + }, + /** + Works like mustThrow() but expects filter to be a regex, + function, or string to match/filter the resulting exception + against. If f() does not throw, this test fails and an Error is + thrown. If filter is a regex, the test passes if + filter.test(error.message) passes. If it's a function, the test + passes if filter(error) returns truthy. If it's a string, the + test passes if the filter matches the exception message + precisely. In all other cases the test fails, throwing an + Error. + + If it throws, msg is used as the error report unless it's falsy, + in which case a default is used. + */ + mustThrowMatching: function(f, filter, msg){ + ++this.counter; + let err; + try{ f(); } catch(e){err=e;} + if(!err) throw new Error(msg || "Expected exception."); + let pass = false; + if(filter instanceof RegExp) pass = filter.test(err.message); + else if(filter instanceof Function) pass = filter(err); + else if('string' === typeof filter) pass = (err.message === filter); + if(!pass){ + throw new Error(msg || ("Filter rejected this exception: "+err.message)); + } + return this; + }, + /** Throws if expr is truthy or expr is a function and expr() + returns truthy. */ + throwIf: function(expr, msg){ + ++this.counter; + if(this.toBool(expr)) throw new Error(msg || "throwIf() failed"); + return this; + }, + /** Throws if expr is falsy or expr is a function and expr() + returns falsy. */ + throwUnless: function(expr, msg){ + ++this.counter; + if(!this.toBool(expr)) throw new Error(msg || "throwUnless() failed"); + return this; + } + }; + + + /** + This is a module object for use with the emscripten-installed + sqlite3InitModule() factory function. + */ + self.sqlite3TestModule = { + postRun: [ + /* function(theModule){...} */ + ], + //onRuntimeInitialized: function(){}, + /* Proxy for C-side stdout output. */ + print: function(){ + console.log.apply(console, Array.prototype.slice.call(arguments)); + }, + /* Proxy for C-side stderr output. */ + printErr: function(){ + console.error.apply(console, Array.prototype.slice.call(arguments)); + }, + /** + Called by the module init bits to report loading + progress. It gets passed an empty argument when loading is + done (after onRuntimeInitialized() and any this.postRun + callbacks have been run). + */ + setStatus: function f(text){ + if(!f.last){ + f.last = { text: '', step: 0 }; + f.ui = { + status: E('#module-status'), + progress: E('#module-progress'), + spinner: E('#module-spinner') + }; + } + if(text === f.last.text) return; + f.last.text = text; + if(f.ui.progress){ + f.ui.progress.value = f.last.step; + f.ui.progress.max = f.last.step + 1; + } + ++f.last.step; + if(text) { + f.ui.status.classList.remove('hidden'); + f.ui.status.innerText = text; + }else{ + if(f.ui.progress){ + f.ui.progress.remove(); + f.ui.spinner.remove(); + delete f.ui.progress; + delete f.ui.spinner; + } + f.ui.status.classList.add('hidden'); + } + } + }; +})(self/*window or worker*/); diff --git a/ext/wasm/common/emscripten.css b/ext/wasm/common/emscripten.css new file mode 100644 index 0000000000..7e3dc811d0 --- /dev/null +++ b/ext/wasm/common/emscripten.css @@ -0,0 +1,24 @@ +/* emcscript-related styling, used during the module load/intialization processes... */ +.emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; } +div.emscripten { text-align: center; } +div.emscripten_border { border: 1px solid black; } +#module-spinner { overflow: visible; } +#module-spinner > * { + margin-top: 1em; +} +.spinner { + height: 50px; + width: 50px; + margin: 0px auto; + animation: rotation 0.8s linear infinite; + border-left: 10px solid rgb(0,150,240); + border-right: 10px solid rgb(0,150,240); + border-bottom: 10px solid rgb(0,150,240); + border-top: 10px solid rgb(100,0,200); + border-radius: 100%; + background-color: rgb(200,100,250); +} +@keyframes rotation { + from {transform: rotate(0deg);} + to {transform: rotate(360deg);} +} diff --git a/ext/fiddle/testing.css b/ext/wasm/common/testing.css similarity index 93% rename from ext/fiddle/testing.css rename to ext/wasm/common/testing.css index f87dbd2cf1..09c570f48a 100644 --- a/ext/fiddle/testing.css +++ b/ext/wasm/common/testing.css @@ -29,3 +29,4 @@ span.labeled-input { color: red; background-color: yellow; } +#test-output { font-family: monospace } diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js new file mode 100644 index 0000000000..5a1d425caf --- /dev/null +++ b/ext/wasm/common/whwasmutil.js @@ -0,0 +1,1548 @@ +/** + 2022-07-08 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + The whwasmutil is developed in conjunction with the Jaccwabyt + project: + + https://fossil.wanderinghorse.net/r/jaccwabyt + + Maintenance reminder: If you're reading this in a tree other than + the Jaccwabyt tree, note that this copy may be replaced with + upstream copies of that one from time to time. Thus the code + installed by this function "should not" be edited outside of that + project, else it risks getting overwritten. +*/ +/** + This function is intended to simplify porting around various bits + of WASM-related utility code from project to project. + + The primary goal of this code is to replace, where possible, + Emscripten-generated glue code with equivalent utility code which + can be used in arbitrary WASM environments built with toolchains + other than Emscripten. As of this writing, this code is capable of + acting as a replacement for Emscripten's generated glue code + _except_ that the latter installs handlers for Emscripten-provided + APIs such as its "FS" (virtual filesystem) API. Loading of such + things still requires using Emscripten's glue, but the post-load + utility APIs provided by this code are still usable as replacements + for their sub-optimally-documented Emscripten counterparts. + + Intended usage: + + ``` + self.WhWasmUtilInstaller(appObject); + delete self.WhWasmUtilInstaller; + ``` + + Its global-scope symbol is intended only to provide an easy way to + make it available to 3rd-party scripts and "should" be deleted + after calling it. That symbols is _not_ used within the library. + + Forewarning: this API explicitly targets only browser + environments. If a given non-browser environment has the + capabilities needed for a given feature (e.g. TextEncoder), great, + but it does not go out of its way to account for them and does not + provide compatibility crutches for them. + + It currently offers alternatives to the following + Emscripten-generated APIs: + + - OPTIONALLY memory allocation, but how this gets imported is + environment-specific. Most of the following features only work + if allocation is available. + + - WASM-exported "indirect function table" access and + manipulation. e.g. creating new WASM-side functions using JS + functions, analog to Emscripten's addFunction() and + removeFunction() but slightly different. + + - Get/set specific heap memory values, analog to Emscripten's + getValue() and setValue(). + + - String length counting in UTF-8 bytes (C-style and JS strings). + + - JS string to C-string conversion and vice versa, analog to + Emscripten's stringToUTF8Array() and friends, but with slighter + different interfaces. + + - JS string to Uint8Array conversion, noting that browsers actually + already have this built in via TextEncoder. + + - "Scoped" allocation, such that allocations made inside of a given + explicit scope will be automatically cleaned up when the scope is + closed. This is fundamentally similar to Emscripten's + stackAlloc() and friends but uses the heap instead of the stack + because access to the stack requires C code. + + - Create JS wrappers for WASM functions, analog to Emscripten's + ccall() and cwrap() functions, except that the automatic + conversions for function arguments and return values can be + easily customized by the client by assigning custom function + signature type names to conversion functions. Essentially, + it's ccall() and cwrap() on steroids. + + How to install... + + Passing an object to this function will install the functionality + into that object. Afterwards, client code "should" delete the global + symbol. + + This code requires that the target object have the following + properties, noting that they needn't be available until the first + time one of the installed APIs is used (as opposed to when this + function is called) except where explicitly noted: + + - `exports` must be a property of the target object OR a property + of `target.instance` (a WebAssembly.Module instance) and it must + contain the symbols exported by the WASM module associated with + this code. In an Enscripten environment it must be set to + `Module['asm']`. The exports object must contain a minimum of the + following symbols: + + - `memory`: a WebAssembly.Memory object representing the WASM + memory. _Alternately_, the `memory` property can be set on the + target instance, in particular if the WASM heap memory is + initialized in JS an _imported_ into WASM, as opposed to being + initialized in WASM and exported to JS. + + - `__indirect_function_table`: the WebAssembly.Table object which + holds WASM-exported functions. This API does not strictly + require that the table be able to grow but it will throw if its + `installFunction()` is called and the table cannot grow. + + In order to simplify downstream usage, if `target.exports` is not + set when this is called then a property access interceptor + (read-only, configurable, enumerable) gets installed as `exports` + which resolves to `target.instance.exports`, noting that the latter + property need not exist until the first time `target.exports` is + accessed. + + Some APIs _optionally_ make use of the `bigIntEnabled` property of + the target object. It "should" be set to true if the WASM + environment is compiled with BigInt support, else it must be + false. If it is false, certain BigInt-related features will trigger + an exception if invoked. This property, if not set when this is + called, will get a default value of true only if the BigInt64Array + constructor is available, else it will default to false. + + Some optional APIs require that the target have the following + methods: + + - 'alloc()` must behave like C's `malloc()`, allocating N bytes of + memory and returning its pointer. In Emscripten this is + conventionally made available via `Module['_malloc']`. This API + requires that the alloc routine throw on allocation error, as + opposed to returning null or 0. + + - 'dealloc()` must behave like C's `free()`, accepting either a + pointer returned from its allocation counterpart or the values + null/0 (for which it must be a no-op). allocating N bytes of + memory and returning its pointer. In Emscripten this is + conventionally made available via `Module['_free']`. + + APIs which require allocation routines are explicitly documented as + such and/or have "alloc" in their names. + + This code is developed and maintained in conjunction with the + Jaccwabyt project: + + https://fossil.wanderinghorse.net/r/jaccwabbyt + + More specifically: + + https://fossil.wanderinghorse.net/r/jaccwabbyt/file/common/whwasmutil.js +*/ +self.WhWasmUtilInstaller = function(target){ + 'use strict'; + if(undefined===target.bigIntEnabled){ + target.bigIntEnabled = !!self['BigInt64Array']; + } + + /** Throws a new Error, the message of which is the concatenation of + all args with a space between each. */ + const toss = (...args)=>{throw new Error(args.join(' '))}; + + if(!target.exports){ + Object.defineProperty(target, 'exports', { + enumerable: true, configurable: true, + get: ()=>(target.instance && target.instance.exports) + }); + } + + /********* + alloc()/dealloc() auto-install... + + This would be convenient but it can also cause us to pick up + malloc() even when the client code is using a different exported + allocator (who, me?), which is bad. malloc() may be exported even + if we're not explicitly using it and overriding the malloc() + function, linking ours first, is not always feasible when using a + malloc() proxy, as it can lead to recursion and stack overflow + (who, me?). So... we really need the downstream code to set up + target.alloc/dealloc() itself. + ******/ + /****** + if(target.exports){ + //Maybe auto-install alloc()/dealloc()... + if(!target.alloc && target.exports.malloc){ + target.alloc = function(n){ + const m = this(n); + return m || toss("Allocation of",n,"byte(s) failed."); + }.bind(target.exports.malloc); + } + + if(!target.dealloc && target.exports.free){ + target.dealloc = function(ptr){ + if(ptr) this(ptr); + }.bind(target.exports.free); + } + }*******/ + + /** + Pointers in WASM are currently assumed to be 32-bit, but someday + that will certainly change. + */ + const ptrIR = target.pointerIR || 'i32'; + const ptrSizeof = ('i32'===ptrIR ? 4 + : ('i64'===ptrIR + ? 8 : toss("Unhandled ptrSizeof:",ptrIR))); + /** Stores various cached state. */ + const cache = Object.create(null); + /** Previously-recorded size of cache.memory.buffer, noted so that + we can recreate the view objects if the heap grows. */ + cache.heapSize = 0; + /** WebAssembly.Memory object extracted from target.memory or + target.exports.memory the first time heapWrappers() is + called. */ + cache.memory = null; + /** uninstallFunction() puts table indexes in here for reuse and + installFunction() extracts them. */ + cache.freeFuncIndexes = []; + /** + Used by scopedAlloc() and friends. + */ + cache.scopedAlloc = []; + + cache.utf8Decoder = new TextDecoder(); + cache.utf8Encoder = new TextEncoder('utf-8'); + + /** + If (cache.heapSize !== cache.memory.buffer.byteLength), i.e. if + the heap has grown since the last call, updates cache.HEAPxyz. + Returns the cache object. + */ + const heapWrappers = function(){ + if(!cache.memory){ + cache.memory = (target.memory instanceof WebAssembly.Memory) + ? target.memory : target.exports.memory; + }else if(cache.heapSize === cache.memory.buffer.byteLength){ + return cache; + } + // heap is newly-acquired or has been resized.... + const b = cache.memory.buffer; + cache.HEAP8 = new Int8Array(b); cache.HEAP8U = new Uint8Array(b); + cache.HEAP16 = new Int16Array(b); cache.HEAP16U = new Uint16Array(b); + cache.HEAP32 = new Int32Array(b); cache.HEAP32U = new Uint32Array(b); + if(target.bigIntEnabled){ + cache.HEAP64 = new BigInt64Array(b); cache.HEAP64U = new BigUint64Array(b); + } + cache.HEAP32F = new Float32Array(b); cache.HEAP64F = new Float64Array(b); + cache.heapSize = b.byteLength; + return cache; + }; + + /** Convenience equivalent of this.heapForSize(8,false). */ + target.heap8 = ()=>heapWrappers().HEAP8; + + /** Convenience equivalent of this.heapForSize(8,true). */ + target.heap8u = ()=>heapWrappers().HEAP8U; + + /** Convenience equivalent of this.heapForSize(16,false). */ + target.heap16 = ()=>heapWrappers().HEAP16; + + /** Convenience equivalent of this.heapForSize(16,true). */ + target.heap16u = ()=>heapWrappers().HEAP16U; + + /** Convenience equivalent of this.heapForSize(32,false). */ + target.heap32 = ()=>heapWrappers().HEAP32; + + /** Convenience equivalent of this.heapForSize(32,true). */ + target.heap32u = ()=>heapWrappers().HEAP32U; + + /** + Requires n to be one of: + + - integer 8, 16, or 32. + - A integer-type TypedArray constructor: Int8Array, Int16Array, + Int32Array, or their Uint counterparts. + + If this.bigIntEnabled is true, it also accepts the value 64 or a + BigInt64Array/BigUint64Array, else it throws if passed 64 or one + of those constructors. + + Returns an integer-based TypedArray view of the WASM heap + memory buffer associated with the given block size. If passed + an integer as the first argument and unsigned is truthy then + the "U" (unsigned) variant of that view is returned, else the + signed variant is returned. If passed a TypedArray value, the + 2nd argument is ignores. Note that Float32Array and + Float64Array views are not supported by this function. + + Note that growth of the heap will invalidate any references to + this heap, so do not hold a reference longer than needed and do + not use a reference after any operation which may + allocate. Instead, re-fetch the reference by calling this + function again. + + Throws if passed an invalid n. + + Pedantic side note: the name "heap" is a bit of a misnomer. In an + Emscripten environment, the memory managed via the stack + allocation API is in the same Memory object as the heap (which + makes sense because otherwise arbitrary pointer X would be + ambiguous: is it in the heap or the stack?). + */ + target.heapForSize = function(n,unsigned = false){ + let ctor; + const c = (cache.memory && cache.heapSize === cache.memory.buffer.byteLength) + ? cache : heapWrappers(); + switch(n){ + case Int8Array: return c.HEAP8; case Uint8Array: return c.HEAP8U; + case Int16Array: return c.HEAP16; case Uint16Array: return c.HEAP16U; + case Int32Array: return c.HEAP32; case Uint32Array: return c.HEAP32U; + case 8: return unsigned ? c.HEAP8U : c.HEAP8; + case 16: return unsigned ? c.HEAP16U : c.HEAP16; + case 32: return unsigned ? c.HEAP32U : c.HEAP32; + case 64: + if(c.HEAP64) return unsigned ? c.HEAP64U : c.HEAP64; + break; + default: + if(this.bigIntEnabled){ + if(n===self['BigUint64Array']) return c.HEAP64U; + else if(n===self['BigInt64Array']) return c.HEAP64; + break; + } + } + toss("Invalid heapForSize() size: expecting 8, 16, 32,", + "or (if BigInt is enabled) 64."); + }.bind(target); + + /** + Returns the WASM-exported "indirect function table." + */ + target.functionTable = function(){ + return target.exports.__indirect_function_table; + /** -----------------^^^^^ "seems" to be a standardized export name. + From Emscripten release notes from 2020-09-10: + - Use `__indirect_function_table` as the import name for the + table, which is what LLVM does. + */ + }.bind(target); + + /** + Given a function pointer, returns the WASM function table entry + if found, else returns a falsy value. + */ + target.functionEntry = function(fptr){ + const ft = this.functionTable(); + return fptr < ft.length ? ft.get(fptr) : undefined; + }.bind(target); + + /** + Creates a WASM function which wraps the given JS function and + returns the JS binding of that WASM function. The signature + argument must be the Jaccwabyt-format or Emscripten + addFunction()-format function signature string. In short: in may + have one of the following formats: + + - Emscripten: `x...`, where the first x is a letter representing + the result type and subsequent letters represent the argument + types. See below. + + - Jaccwabyt: `x(...)` where `x` is the letter representing the + result type and letters in the parens (if any) represent the + argument types. See below. + + Supported letters: + + - `i` = int32 + - `p` = int32 ("pointer") + - `j` = int64 + - `f` = float32 + - `d` = float64 + - `v` = void, only legal for use as the result type + + It throws if an invalid signature letter is used. + + Jaccwabyt-format signatures support some additional letters which + have no special meaning here but (in this context) act as aliases + for other letters: + + - `s`, `P`: same as `p` + + Sidebar: this code is developed together with Jaccwabyt, thus the + support for its signature format. + */ + target.jsFuncToWasm = function f(func, sig){ + /** Attribution: adapted up from Emscripten-generated glue code, + refactored primarily for efficiency's sake, eliminating + call-local functions and superfluous temporary arrays. */ + if(!f._){/*static init...*/ + f._ = { + // Map of signature letters to type IR values + sigTypes: Object.create(null), + // Map of type IR values to WASM type code values + typeCodes: Object.create(null), + /** Encodes n, which must be <2^14 (16384), into target array + tgt, as a little-endian value, using the given method + ('push' or 'unshift'). */ + uleb128Encode: function(tgt, method, n){ + if(n<128) tgt[method](n); + else tgt[method]( (n % 128) | 128, n>>7); + }, + /** Intentionally-lax pattern for Jaccwabyt-format function + pointer signatures, the intent of which is simply to + distinguish them from Emscripten-format signatures. The + downstream checks are less lax. */ + rxJSig: /^(\w)\((\w*)\)$/, + /** Returns the parameter-value part of the given signature + string. */ + sigParams: function(sig){ + const m = f._.rxJSig.exec(sig); + return m ? m[2] : sig.substr(1); + }, + /** Returns the IR value for the given letter or throws + if the letter is invalid. */ + letterType: (x)=>f._.sigTypes[x] || toss("Invalid signature letter:",x), + /** Returns an object describing the result type and parameter + type(s) of the given function signature, or throws if the + signature is invalid. */ + /******** // only valid for use with the WebAssembly.Function ctor, which + // is not yet documented on MDN. + sigToWasm: function(sig){ + const rc = {parameters:[], results: []}; + if('v'!==sig[0]) rc.results.push(f._.letterType(sig[0])); + for(const x of f._.sigParams(sig)){ + rc.parameters.push(f._.letterType(x)); + } + return rc; + },************/ + /** Pushes the WASM data type code for the given signature + letter to the given target array. Throws if letter is + invalid. */ + pushSigType: (dest, letter)=>dest.push(f._.typeCodes[f._.letterType(letter)]) + }; + f._.sigTypes.i = f._.sigTypes.p = f._.sigTypes.P = f._.sigTypes.s = 'i32'; + f._.sigTypes.j = 'i64'; f._.sigTypes.f = 'f32'; f._.sigTypes.d = 'f64'; + f._.typeCodes['i32'] = 0x7f; f._.typeCodes['i64'] = 0x7e; + f._.typeCodes['f32'] = 0x7d; f._.typeCodes['f64'] = 0x7c; + }/*static init*/ + const sigParams = f._.sigParams(sig); + const wasmCode = [0x01/*count: 1*/, 0x60/*function*/]; + f._.uleb128Encode(wasmCode, 'push', sigParams.length); + for(const x of sigParams) f._.pushSigType(wasmCode, x); + if('v'===sig[0]) wasmCode.push(0); + else{ + wasmCode.push(1); + f._.pushSigType(wasmCode, sig[0]); + } + f._.uleb128Encode(wasmCode, 'unshift', wasmCode.length)/* type section length */; + wasmCode.unshift( + 0x00, 0x61, 0x73, 0x6d, /* magic: "\0asm" */ + 0x01, 0x00, 0x00, 0x00, /* version: 1 */ + 0x01 /* type section code */ + ); + wasmCode.push( + /* import section: */ 0x02, 0x07, + /* (import "e" "f" (func 0 (type 0))): */ + 0x01, 0x01, 0x65, 0x01, 0x66, 0x00, 0x00, + /* export section: */ 0x07, 0x05, + /* (export "f" (func 0 (type 0))): */ + 0x01, 0x01, 0x66, 0x00, 0x00 + ); + return (new WebAssembly.Instance( + new WebAssembly.Module(new Uint8Array(wasmCode)), { + e: { f: func } + })).exports['f']; + }/*jsFuncToWasm()*/; + + /** + Expects a JS function and signature, exactly as for + this.jsFuncToWasm(). It uses that function to create a + WASM-exported function, installs that function to the next + available slot of this.functionTable(), and returns the + function's index in that table (which acts as a pointer to that + function). The returned pointer can be passed to + removeFunction() to uninstall it and free up the table slot for + reuse. + + As a special case, if the passed-in function is a WASM-exported + function then the signature argument is ignored and func is + installed as-is, without requiring re-compilation/re-wrapping. + + This function will propagate an exception if + WebAssembly.Table.grow() throws or this.jsFuncToWasm() throws. + The former case can happen in an Emscripten-compiled + environment when building without Emscripten's + `-sALLOW_TABLE_GROWTH` flag. + + Sidebar: this function differs from Emscripten's addFunction() + _primarily_ in that it does not share that function's + undocumented behavior of reusing a function if it's passed to + addFunction() more than once, which leads to removeFunction() + breaking clients which do not take care to avoid that case: + + https://github.com/emscripten-core/emscripten/issues/17323 + */ + target.installFunction = function f(func, sig){ + const ft = this.functionTable(); + const oldLen = ft.length; + let ptr; + while(cache.freeFuncIndexes.length){ + ptr = cache.freeFuncIndexes.pop(); + if(ft.get(ptr)){ /* Table was modified via a different API */ + ptr = null; + continue; + }else{ + break; + } + } + if(!ptr){ + ptr = oldLen; + ft.grow(1); + } + try{ + /*this will only work if func is a WASM-exported function*/ + ft.set(ptr, func); + return ptr; + }catch(e){ + if(!(e instanceof TypeError)){ + if(ptr===oldLen) cache.freeFuncIndexes.push(oldLen); + throw e; + } + } + // It's not a WASM-exported function, so compile one... + try { + ft.set(ptr, this.jsFuncToWasm(func, sig)); + }catch(e){ + if(ptr===oldLen) cache.freeFuncIndexes.push(oldLen); + throw e; + } + return ptr; + }.bind(target); + + /** + Requires a pointer value previously returned from + this.installFunction(). Removes that function from the WASM + function table, marks its table slot as free for re-use, and + returns that function. It is illegal to call this before + installFunction() has been called and results are undefined if + ptr was not returned by that function. The returned function + may be passed back to installFunction() to reinstall it. + */ + target.uninstallFunction = function(ptr){ + const fi = cache.freeFuncIndexes; + const ft = this.functionTable(); + fi.push(ptr); + const rc = ft.get(ptr); + ft.set(ptr, null); + return rc; + }.bind(target); + + /** + Given a WASM heap memory address and a data type name in the form + (i8, i16, i32, i64, float (or f32), double (or f64)), this + fetches the numeric value from that address and returns it as a + number or, for the case of type='i64', a BigInt (noting that that + type triggers an exception if this.bigIntEnabled is + falsy). Throws if given an invalid type. + + As a special case, if type ends with a `*`, it is considered to + be a pointer type and is treated as the WASM numeric type + appropriate for the pointer size (`i32`). + + While likely not obvious, this routine and its setMemValue() + counterpart are how pointer-to-value _output_ parameters + in WASM-compiled C code can be interacted with: + + ``` + const ptr = alloc(4); + setMemValue(ptr, 0, 'i32'); // clear the ptr's value + aCFuncWithOutputPtrToInt32Arg( ptr ); // e.g. void foo(int *x); + const result = getMemValue(ptr, 'i32'); // fetch ptr's value + dealloc(ptr); + ``` + + scopedAlloc() and friends can be used to make handling of + `ptr` safe against leaks in the case of an exception: + + ``` + let result; + const scope = scopedAllocPush(); + try{ + const ptr = scopedAlloc(4); + setMemValue(ptr, 0, 'i32'); + aCFuncWithOutputPtrArg( ptr ); + result = getMemValue(ptr, 'i32'); + }finally{ + scopedAllocPop(scope); + } + ``` + + As a rule setMemValue() must be called to set (typically zero + out) the pointer's value, else it will contain an essentially + random value. + + See: setMemValue() + */ + target.getMemValue = function(ptr, type='i8'){ + if(type.endsWith('*')) type = ptrIR; + const c = (cache.memory && cache.heapSize === cache.memory.buffer.byteLength) + ? cache : heapWrappers(); + switch(type){ + case 'i1': + case 'i8': return c.HEAP8[ptr>>0]; + case 'i16': return c.HEAP16[ptr>>1]; + case 'i32': return c.HEAP32[ptr>>2]; + case 'i64': + if(this.bigIntEnabled) return BigInt(c.HEAP64[ptr>>3]); + break; + case 'float': case 'f32': return c.HEAP32F[ptr>>2]; + case 'double': case 'f64': return Number(c.HEAP64F[ptr>>3]); + default: break; + } + toss('Invalid type for getMemValue():',type); + }.bind(target); + + /** + The counterpart of getMemValue(), this sets a numeric value at + the given WASM heap address, using the type to define how many + bytes are written. Throws if given an invalid type. See + getMemValue() for details about the type argument. If the 3rd + argument ends with `*` then it is treated as a pointer type and + this function behaves as if the 3rd argument were `i32`. + + This function returns itself. + */ + target.setMemValue = function f(ptr, value, type='i8'){ + if (type.endsWith('*')) type = ptrIR; + const c = (cache.memory && cache.heapSize === cache.memory.buffer.byteLength) + ? cache : heapWrappers(); + switch (type) { + case 'i1': + case 'i8': c.HEAP8[ptr>>0] = value; return f; + case 'i16': c.HEAP16[ptr>>1] = value; return f; + case 'i32': c.HEAP32[ptr>>2] = value; return f; + case 'i64': + if(c.HEAP64){ + c.HEAP64[ptr>>3] = BigInt(value); + return f; + } + break; + case 'float': case 'f32': c.HEAP32F[ptr>>2] = value; return f; + case 'double': case 'f64': c.HEAP64F[ptr>>3] = value; return f; + } + toss('Invalid type for setMemValue(): ' + type); + }; + + /** + Expects ptr to be a pointer into the WASM heap memory which + refers to a NUL-terminated C-style string encoded as UTF-8. + Returns the length, in bytes, of the string, as for `strlen(3)`. + As a special case, if !ptr then it it returns `null`. Throws if + ptr is out of range for target.heap8u(). + */ + target.cstrlen = function(ptr){ + if(!ptr) return null; + const h = heapWrappers().HEAP8U; + let pos = ptr; + for( ; h[pos] !== 0; ++pos ){} + return pos - ptr; + }; + + /** + Expects ptr to be a pointer into the WASM heap memory which + refers to a NUL-terminated C-style string encoded as UTF-8. This + function counts its byte length using cstrlen() then returns a + JS-format string representing its contents. As a special case, if + ptr is falsy, `null` is returned. + */ + target.cstringToJs = function(ptr){ + const n = this.cstrlen(ptr); + if(null===n) return n; + return n + ? cache.utf8Decoder.decode( + new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n) + ) : ""; + }.bind(target); + + /** + Given a JS string, this function returns its UTF-8 length in + bytes. Returns null if str is not a string. + */ + target.jstrlen = function(str){ + /** Attribution: derived from Emscripten's lengthBytesUTF8() */ + if('string'!==typeof str) return null; + const n = str.length; + let len = 0; + for(let i = 0; i < n; ++i){ + let u = str.charCodeAt(i); + if(u>=0xd800 && u<=0xdfff){ + u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt(++i) & 0x3FF); + } + if(u<=0x7f) ++len; + else if(u<=0x7ff) len += 2; + else if(u<=0xffff) len += 3; + else len += 4; + } + return len; + }; + + /** + Encodes the given JS string as UTF8 into the given TypedArray + tgt, starting at the given offset and writing, at most, maxBytes + bytes (including the NUL terminator if addNul is true, else no + NUL is added). If it writes any bytes at all and addNul is true, + it always NUL-terminates the output, even if doing so means that + the NUL byte is all that it writes. + + If maxBytes is negative (the default) then it is treated as the + remaining length of tgt, starting at the given offset. + + If writing the last character would surpass the maxBytes count + because the character is multi-byte, that character will not be + written (as opposed to writing a truncated multi-byte character). + This can lead to it writing as many as 3 fewer bytes than + maxBytes specifies. + + Returns the number of bytes written to the target, _including_ + the NUL terminator (if any). If it returns 0, it wrote nothing at + all, which can happen if: + + - str is empty and addNul is false. + - offset < 0. + - maxBytes == 0. + - maxBytes is less than the byte length of a multi-byte str[0]. + + Throws if tgt is not an Int8Array or Uint8Array. + + Design notes: + + - In C's strcpy(), the destination pointer is the first + argument. That is not the case here primarily because the 3rd+ + arguments are all referring to the destination, so it seems to + make sense to have them grouped with it. + + - Emscripten's counterpart of this function (stringToUTF8Array()) + returns the number of bytes written sans NUL terminator. That + is, however, ambiguous: str.length===0 or maxBytes===(0 or 1) + all cause 0 to be returned. + */ + target.jstrcpy = function(jstr, tgt, offset = 0, maxBytes = -1, addNul = true){ + /** Attribution: the encoding bits are taken from Emscripten's + stringToUTF8Array(). */ + if(!tgt || (!(tgt instanceof Int8Array) && !(tgt instanceof Uint8Array))){ + toss("jstrcpy() target must be an Int8Array or Uint8Array."); + } + if(maxBytes<0) maxBytes = tgt.length - offset; + if(!(maxBytes>0) || !(offset>=0)) return 0; + let i = 0, max = jstr.length; + const begin = offset, end = offset + maxBytes - (addNul ? 1 : 0); + for(; i < max && offset < end; ++i){ + let u = jstr.charCodeAt(i); + if(u>=0xd800 && u<=0xdfff){ + u = 0x10000 + ((u & 0x3FF) << 10) | (jstr.charCodeAt(++i) & 0x3FF); + } + if(u<=0x7f){ + if(offset >= end) break; + tgt[offset++] = u; + }else if(u<=0x7ff){ + if(offset + 1 >= end) break; + tgt[offset++] = 0xC0 | (u >> 6); + tgt[offset++] = 0x80 | (u & 0x3f); + }else if(u<=0xffff){ + if(offset + 2 >= end) break; + tgt[offset++] = 0xe0 | (u >> 12); + tgt[offset++] = 0x80 | ((u >> 6) & 0x3f); + tgt[offset++] = 0x80 | (u & 0x3f); + }else{ + if(offset + 3 >= end) break; + tgt[offset++] = 0xf0 | (u >> 18); + tgt[offset++] = 0x80 | ((u >> 12) & 0x3f); + tgt[offset++] = 0x80 | ((u >> 6) & 0x3f); + tgt[offset++] = 0x80 | (u & 0x3f); + } + } + if(addNul) tgt[offset++] = 0; + return offset - begin; + }; + + /** + Works similarly to C's strncpy(), copying, at most, n bytes (not + characters) from srcPtr to tgtPtr. It copies until n bytes have + been copied or a 0 byte is reached in src. _Unlike_ strncpy(), it + returns the number of bytes it assigns in tgtPtr, _including_ the + NUL byte (if any). If n is reached before a NUL byte in srcPtr, + tgtPtr will _not_ be NULL-terminated. If a NUL byte is reached + before n bytes are copied, tgtPtr will be NUL-terminated. + + If n is negative, cstrlen(srcPtr)+1 is used to calculate it, the + +1 being for the NUL byte. + + Throws if tgtPtr or srcPtr are falsy. Results are undefined if: + + - either is not a pointer into the WASM heap or + + - srcPtr is not NUL-terminated AND n is less than srcPtr's + logical length. + + ACHTUNG: it is possible to copy partial multi-byte characters + this way, and converting such strings back to JS strings will + have undefined results. + */ + target.cstrncpy = function(tgtPtr, srcPtr, n){ + if(!tgtPtr || !srcPtr) toss("cstrncpy() does not accept NULL strings."); + if(n<0) n = this.cstrlen(strPtr)+1; + else if(!(n>0)) return 0; + const heap = this.heap8u(); + let i = 0, ch; + for(; i < n && (ch = heap[srcPtr+i]); ++i){ + heap[tgtPtr+i] = ch; + } + if(i{ + return cache.utf8Encoder.encode(addNul ? (str+"\0") : str); + // Or the hard way... + /** Attribution: derived from Emscripten's stringToUTF8Array() */ + //const a = [], max = str.length; + //let i = 0, pos = 0; + //for(; i < max; ++i){ + // let u = str.charCodeAt(i); + // if(u>=0xd800 && u<=0xdfff){ + // u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt(++i) & 0x3FF); + // } + // if(u<=0x7f) a[pos++] = u; + // else if(u<=0x7ff){ + // a[pos++] = 0xC0 | (u >> 6); + // a[pos++] = 0x80 | (u & 63); + // }else if(u<=0xffff){ + // a[pos++] = 0xe0 | (u >> 12); + // a[pos++] = 0x80 | ((u >> 6) & 63); + // a[pos++] = 0x80 | (u & 63); + // }else{ + // a[pos++] = 0xf0 | (u >> 18); + // a[pos++] = 0x80 | ((u >> 12) & 63); + // a[pos++] = 0x80 | ((u >> 6) & 63); + // a[pos++] = 0x80 | (u & 63); + // } + // } + // return new Uint8Array(a); + }; + + const __affirmAlloc = (obj,funcName)=>{ + if(!(obj.alloc instanceof Function) || + !(obj.dealloc instanceof Function)){ + toss("Object is missing alloc() and/or dealloc() function(s)", + "required by",funcName+"()."); + } + }; + + const __allocCStr = function(jstr, returnWithLength, allocator, funcName){ + __affirmAlloc(this, funcName); + if('string'!==typeof jstr) return null; + const n = this.jstrlen(jstr), + ptr = allocator(n+1); + this.jstrcpy(jstr, this.heap8u(), ptr, n+1, true); + return returnWithLength ? [ptr, n] : ptr; + }.bind(target); + + /** + Uses target.alloc() to allocate enough memory for jstrlen(jstr)+1 + bytes of memory, copies jstr to that memory using jstrcpy(), + NUL-terminates it, and returns the pointer to that C-string. + Ownership of the pointer is transfered to the caller, who must + eventually pass the pointer to dealloc() to free it. + + If passed a truthy 2nd argument then its return semantics change: + it returns [ptr,n], where ptr is the C-string's pointer and n is + its cstrlen(). + + Throws if `target.alloc` or `target.dealloc` are not functions. + */ + target.allocCString = + (jstr, returnWithLength=false)=>__allocCStr(jstr, returnWithLength, + target.alloc, 'allocCString()'); + + /** + Starts an "allocation scope." All allocations made using + scopedAlloc() are recorded in this scope and are freed when the + value returned from this function is passed to + scopedAllocPop(). + + This family of functions requires that the API's object have both + `alloc()` and `dealloc()` methods, else this function will throw. + + Intended usage: + + ``` + const scope = scopedAllocPush(); + try { + const ptr1 = scopedAlloc(100); + const ptr2 = scopedAlloc(200); + const ptr3 = scopedAlloc(300); + ... + // Note that only allocations made via scopedAlloc() + // are managed by this allocation scope. + }finally{ + scopedAllocPop(scope); + } + ``` + + The value returned by this function must be treated as opaque by + the caller, suitable _only_ for passing to scopedAllocPop(). + Its type and value are not part of this function's API and may + change in any given version of this code. + + `scopedAlloc.level` can be used to determine how many scoped + alloc levels are currently active. + */ + target.scopedAllocPush = function(){ + __affirmAlloc(this, 'scopedAllocPush'); + const a = []; + cache.scopedAlloc.push(a); + return a; + }.bind(target); + + /** + Cleans up all allocations made using scopedAlloc() in the context + of the given opaque state object, which must be a value returned + by scopedAllocPush(). See that function for an example of how to + use this function. + + Though scoped allocations are managed like a stack, this API + behaves properly if allocation scopes are popped in an order + other than the order they were pushed. + + If called with no arguments, it pops the most recent + scopedAllocPush() result: + + ``` + scopedAllocPush(); + try{ ... } finally { scopedAllocPop(); } + ``` + + It's generally recommended that it be passed an explicit argument + to help ensure that push/push are used in matching pairs, but in + trivial code that may be a non-issue. + */ + target.scopedAllocPop = function(state){ + __affirmAlloc(this, 'scopedAllocPop'); + const n = arguments.length + ? cache.scopedAlloc.indexOf(state) + : cache.scopedAlloc.length-1; + if(n<0) toss("Invalid state object for scopedAllocPop()."); + if(0===arguments.length) state = cache.scopedAlloc[n]; + cache.scopedAlloc.splice(n,1); + for(let p; (p = state.pop()); ) this.dealloc(p); + }.bind(target); + + /** + Allocates n bytes of memory using this.alloc() and records that + fact in the state for the most recent call of scopedAllocPush(). + Ownership of the memory is given to scopedAllocPop(), which + will clean it up when it is called. The memory _must not_ be + passed to this.dealloc(). Throws if this API object is missing + the required `alloc()` or `dealloc()` functions or no scoped + alloc is active. + + See scopedAllocPush() for an example of how to use this function. + + The `level` property of this function can be queried to query how + many scoped allocation levels are currently active. + + See also: scopedAllocPtr(), scopedAllocCString() + */ + target.scopedAlloc = function(n){ + if(!cache.scopedAlloc.length){ + toss("No scopedAllocPush() scope is active."); + } + const p = this.alloc(n); + cache.scopedAlloc[cache.scopedAlloc.length-1].push(p); + return p; + }.bind(target); + + Object.defineProperty(target.scopedAlloc, 'level', { + configurable: false, enumerable: false, + get: ()=>cache.scopedAlloc.length, + set: ()=>toss("The 'active' property is read-only.") + }); + + /** + Works identically to allocCString() except that it allocates the + memory using scopedAlloc(). + + Will throw if no scopedAllocPush() call is active. + */ + target.scopedAllocCString = + (jstr, returnWithLength=false)=>__allocCStr(jstr, returnWithLength, + target.scopedAlloc, 'scopedAllocCString()'); + + /** + Wraps function call func() in a scopedAllocPush() and + scopedAllocPop() block, such that all calls to scopedAlloc() and + friends from within that call will have their memory freed + automatically when func() returns. If func throws or propagates + an exception, the scope is still popped, otherwise it returns the + result of calling func(). + */ + target.scopedAllocCall = function(func){ + this.scopedAllocPush(); + try{ return func() } finally{ this.scopedAllocPop() } + }.bind(target); + + /** Internal impl for allocPtr() and scopedAllocPtr(). */ + const __allocPtr = function(howMany, method){ + __affirmAlloc(this, method); + let m = this[method](howMany * ptrSizeof); + this.setMemValue(m, 0, ptrIR) + if(1===howMany){ + return m; + } + const a = [m]; + for(let i = 1; i < howMany; ++i){ + m += ptrSizeof; + a[i] = m; + this.setMemValue(m, 0, ptrIR); + } + return a; + }.bind(target); + + /** + Allocates a single chunk of memory capable of holding `howMany` + pointers and zeroes them out. If `howMany` is 1 then the memory + chunk is returned directly, else an array of pointer addresses is + returned, which can optionally be used with "destructuring + assignment" like this: + + ``` + const [p1, p2, p3] = allocPtr(3); + ``` + + ACHTUNG: when freeing the memory, pass only the _first_ result + value to dealloc(). The others are part of the same memory chunk + and must not be freed separately. + */ + target.allocPtr = (howMany=1)=>__allocPtr(howMany, 'alloc'); + + /** + Identical to allocPtr() except that it allocates using scopedAlloc() + instead of alloc(). + */ + target.scopedAllocPtr = (howMany=1)=>__allocPtr(howMany, 'scopedAlloc'); + + /** + If target.exports[name] exists, it is returned, else an + exception is thrown. + */ + target.xGet = function(name){ + return target.exports[name] || toss("Cannot find exported symbol:",name); + }; + + const __argcMismatch = + (f,n)=>toss(f+"() requires",n,"argument(s)."); + + /** + Looks up a WASM-exported function named fname from + target.exports. If found, it is called, passed all remaining + arguments, and its return value is returned to xCall's caller. If + not found, an exception is thrown. This function does no + conversion of argument or return types, but see xWrap() + and xCallWrapped() for variants which do. + + As a special case, if passed only 1 argument after the name and + that argument in an Array, that array's entries become the + function arguments. (This is not an ambiguous case because it's + not legal to pass an Array object to a WASM function.) + */ + target.xCall = function(fname, ...args){ + const f = this.xGet(fname); + if(!(f instanceof Function)) toss("Exported symbol",fname,"is not a function."); + if(f.length!==args.length) __argcMismatch(fname,f.length) + /* This is arguably over-pedantic but we want to help clients keep + from shooting themselves in the foot when calling C APIs. */; + return (2===arguments.length && Array.isArray(arguments[1])) + ? f.apply(null, arguments[1]) + : f.apply(null, args); + }.bind(target); + + /** + State for use with xWrap() + */ + cache.xWrap = Object.create(null); + const xcv = cache.xWrap.convert = Object.create(null); + /** Map of type names to argument conversion functions. */ + cache.xWrap.convert.arg = Object.create(null); + /** Map of type names to return result conversion functions. */ + cache.xWrap.convert.result = Object.create(null); + + xcv.arg.i64 = (i)=>BigInt(i); + xcv.arg.i32 = (i)=>(i | 0); + xcv.arg.i16 = (i)=>((i | 0) & 0xFFFF); + xcv.arg.i8 = (i)=>((i | 0) & 0xFF); + xcv.arg.f32 = xcv.arg.float = (i)=>Number(i).valueOf(); + xcv.arg.f64 = xcv.arg.double = xcv.arg.f32; + xcv.arg.int = xcv.arg.i32; + xcv.result['*'] = xcv.result['pointer'] = xcv.arg[ptrIR]; + + for(const t of ['i8', 'i16', 'i32', 'int', 'i64', + 'f32', 'float', 'f64', 'double']){ + xcv.arg[t+'*'] = xcv.result[t+'*'] = xcv.arg[ptrIR] + xcv.result[t] = xcv.arg[t] || toss("Missing arg converter:",t); + } + xcv.arg['**'] = xcv.arg[ptrIR]; + + /** + In order for args of type string to work in various contexts in + the sqlite3 API, we need to pass them on as, variably, a C-string + or a pointer value. Thus for ARGs of type 'string' and + '*'/'pointer' we behave differently depending on whether the + argument is a string or not: + + - If v is a string, scopeAlloc() a new C-string from it and return + that temp string's pointer. + + - Else return the value from the arg adaptor defined for ptrIR. + + TODO? Permit an Int8Array/Uint8Array and convert it to a string? + Would that be too much magic concentrated in one place, ready to + backfire? + */ + xcv.arg.string = xcv.arg['pointer'] = xcv.arg['*'] = function(v){ + if('string'===typeof v) return target.scopedAllocCString(v); + return v ? xcv.arg[ptrIR](v) : null; + }; + xcv.result.string = (i)=>target.cstringToJs(i); + xcv.result['string:free'] = function(i){ + try { return i ? target.cstringToJs(i) : null } + finally{ target.dealloc(i) } + }; + xcv.result.json = (i)=>JSON.parse(target.cstringToJs(i)); + xcv.result['json:free'] = function(i){ + try{ return i ? JSON.parse(target.cstringToJs(i)) : null } + finally{ target.dealloc(i) } + } + xcv.result['void'] = (v)=>undefined; + xcv.result['null'] = (v)=>v; + + if(0){ + /*** + This idea can't currently work because we don't know the + signature for the func and don't have a way for the user to + convey it. To do this we likely need to be able to match + arg/result handlers by a regex, but that would incur an O(N) + cost as we check the regex one at a time. Another use case for + such a thing would be pseudotypes like "int:-1" to say that + the value will always be treated like -1 (which has a useful + case in the sqlite3 bindings). + */ + xcv.arg['func-ptr'] = function(v){ + if(!(v instanceof Function)) return xcv.arg[ptrIR]; + const f = this.jsFuncToWasm(v, WHAT_SIGNATURE); + }.bind(target); + } + + const __xArgAdapter = + (t)=>xcv.arg[t] || toss("Argument adapter not found:",t); + + const __xResultAdapter = + (t)=>xcv.result[t] || toss("Result adapter not found:",t); + + cache.xWrap.convertArg = (t,v)=>__xArgAdapter(t)(v); + cache.xWrap.convertResult = + (t,v)=>(null===t ? v : (t ? __xResultAdapter(t)(v) : undefined)); + + /** + Creates a wrapper for the WASM-exported function fname. Uses + xGet() to fetch the exported function (which throws on + error) and returns either that function or a wrapper for that + function which converts the JS-side argument types into WASM-side + types and converts the result type. If the function takes no + arguments and resultType is `null` then the function is returned + as-is, else a wrapper is created for it to adapt its arguments + and result value, as described below. + + (If you're familiar with Emscripten's ccall() and cwrap(), this + function is essentially cwrap() on steroids.) + + This function's arguments are: + + - fname: the exported function's name. xGet() is used to fetch + this, so will throw if no exported function is found with that + name. + + - resultType: the name of the result type. A literal `null` means + to return the original function's value as-is (mnemonic: there + is "null" conversion going on). Literal `undefined` or the + string `"void"` mean to ignore the function's result and return + `undefined`. Aside from those two special cases, it may be one + of the values described below or any mapping installed by the + client using xWrap.resultAdapter(). + + If passed 3 arguments and the final one is an array, that array + must contain a list of type names (see below) for adapting the + arguments from JS to WASM. If passed 2 arguments, more than 3, + or the 3rd is not an array, all arguments after the 2nd (if any) + are treated as type names. i.e.: + + ``` + xWrap('funcname', 'i32', 'string', 'f64'); + // is equivalent to: + xWrap('funcname', 'i32', ['string', 'f64']); + ``` + + Type names are symbolic names which map the arguments to an + adapter function to convert, if needed, the value before passing + it on to WASM or to convert a return result from WASM. The list + of built-in names: + + - `i8`, `i16`, `i32` (args and results): all integer conversions + which convert their argument to an integer and truncate it to + the given bit length. + + - `N*` (args): a type name in the form `N*`, where N is a numeric + type name, is treated the same as WASM pointer. + + - `*` and `pointer` (args): have multple semantics. They + behave exactly as described below for `string` args. + + - `*` and `pointer` (results): are aliases for the current + WASM pointer numeric type. + + - `**` (args): is simply a descriptive alias for the WASM pointer + type. It's primarily intended to mark output-pointer arguments. + + - `i64` (args and results): passes the value to BigInt() to + convert it to an int64. + + - `f32` (`float`), `f64` (`double`) (args and results): pass + their argument to Number(). i.e. the adaptor does not currently + distinguish between the two types of floating-point numbers. + + Non-numeric conversions include: + + - `string` (args): has two different semantics in order to + accommodate various uses of certain C APIs (e.g. output-style + strings)... + + - If the arg is a string, it creates a _temporary_ C-string to + pass to the exported function, cleaning it up before the + wrapper returns. If a long-lived C-string pointer is + required, that requires client-side code to create the + string, then pass its pointer to the function. + + - Else the arg is assumed to be a pointer to a string the + client has already allocated and it's passed on as + a WASM pointer. + + - `string` (results): treats the result value as a const C-string, + copies it to a JS string, and returns that JS string. + + - `string:free` (results): treats the result value as a non-const + C-string, ownership of which has just been transfered to the + caller. It copies the C-string to a JS string, frees the + C-string, and returns the JS string. If such a result value is + NULL, the JS result is `null`. + + - `json` (results): treats the result as a const C-string and + returns the result of passing the converted-to-JS string to + JSON.parse(). Returns `null` if the C-string is a NULL pointer. + + - `json:free` (results): works exactly like `string:free` but + returns the same thing as the `json` adapter. + + The type names for results and arguments are validated when + xWrap() is called and any unknown names will trigger an + exception. + + Clients may map their own result and argument adapters using + xWrap.resultAdapter() and xWrap.argAdaptor(), noting that not all + type conversions are valid for both arguments _and_ result types + as they often have different memory ownership requirements. + + TODOs: + + - Figure out how/whether we can (semi-)transparently handle + pointer-type _output_ arguments. Those currently require + explicit handling by allocating pointers, assigning them before + the call using setMemValue(), and fetching them with + getMemValue() after the call. We may be able to automate some + or all of that. + + - Figure out whether it makes sense to extend the arg adapter + interface such that each arg adapter gets an array containing + the results of the previous arguments in the current call. That + might allow some interesting type-conversion feature. Use case: + handling of the final argument to sqlite3_prepare_v2() depends + on the type (pointer vs JS string) of its 2nd + argument. Currently that distinction requires hand-writing a + wrapper for that function. That case is unusual enough that + abstracting it into this API (and taking on the associated + costs) may well not make good sense. + */ + target.xWrap = function(fname, resultType, ...argTypes){ + if(3===arguments.length && Array.isArray(arguments[2])){ + argTypes = arguments[2]; + } + const xf = this.xGet(fname); + if(argTypes.length!==xf.length) __argcMismatch(fname, xf.length) + if((null===resultType) && 0===xf.length){ + /* Func taking no args with an as-is return. We don't need a wrapper. */ + return xf; + } + /*Verify the arg type conversions are valid...*/; + if(undefined!==resultType && null!==resultType) __xResultAdapter(resultType); + argTypes.forEach(__xArgAdapter) + if(0===xf.length){ + // No args to convert, so we can create a simpler wrapper... + return function(){ + return (arguments.length + ? __argcMismatch(fname, xf.length) + : cache.xWrap.convertResult(resultType, xf.call(null))); + }; + } + return function(...args){ + if(args.length!==xf.length) __argcMismatch(fname, xf.length); + const scope = this.scopedAllocPush(); + try{ + const rc = xf.apply(null,args.map((v,i)=>cache.xWrap.convertArg(argTypes[i], v))); + return cache.xWrap.convertResult(resultType, rc); + }finally{ + this.scopedAllocPop(scope); + } + }.bind(this); + }.bind(target)/*xWrap()*/; + + /** Internal impl for xWrap.resultAdapter() and argAdaptor(). */ + const __xAdapter = function(func, argc, typeName, adapter, modeName, xcvPart){ + if('string'===typeof typeName){ + if(1===argc) return xcvPart[typeName]; + else if(2===argc){ + if(!adapter){ + delete xcvPart[typeName]; + return func; + }else if(!(adapter instanceof Function)){ + toss(modeName,"requires a function argument."); + } + xcvPart[typeName] = adapter; + return func; + } + } + toss("Invalid arguments to",modeName); + }; + + /** + Gets, sets, or removes a result value adapter for use with + xWrap(). If passed only 1 argument, the adapter function for the + given type name is returned. If the second argument is explicit + falsy (as opposed to defaulted), the adapter named by the first + argument is removed. If the 2nd argument is not falsy, it must be + a function which takes one value and returns a value appropriate + for the given type name. The adapter may throw if its argument is + not of a type it can work with. This function throws for invalid + arguments. + + Example: + + ``` + xWrap.resultAdapter('twice',(v)=>v+v); + ``` + + xWrap.resultAdapter() MUST NOT use the scopedAlloc() family of + APIs to allocate a result value. xWrap()-generated wrappers run + in the context of scopedAllocPush() so that argument adapters can + easily convert, e.g., to C-strings, and have them cleaned up + automatically before the wrapper returns to the caller. Likewise, + if a _result_ adapter uses scoped allocation, the result will be + freed before because they would be freed before the wrapper + returns, leading to chaos and undefined behavior. + + Except when called as a getter, this function returns itself. + */ + target.xWrap.resultAdapter = function f(typeName, adapter){ + return __xAdapter(f, arguments.length, typeName, adapter, + 'resultAdaptor()', xcv.result); + }; + + /** + Functions identically to xWrap.resultAdapter() but applies to + call argument conversions instead of result value conversions. + + xWrap()-generated wrappers perform argument conversion in the + context of a scopedAllocPush(), so any memory allocation + performed by argument adapters really, really, really should be + made using the scopedAlloc() family of functions unless + specifically necessary. For example: + + ``` + xWrap.argAdapter('my-string', function(v){ + return ('string'===typeof v) + ? myWasmObj.scopedAllocCString(v) : null; + }; + ``` + + Contrariwise, xWrap.resultAdapter() must _not_ use scopedAlloc() + to allocate its results because they would be freed before the + xWrap()-created wrapper returns. + + Note that it is perfectly legitimate to use these adapters to + perform argument validation, as opposed (or in addition) to + conversion. + */ + target.xWrap.argAdapter = function f(typeName, adapter){ + return __xAdapter(f, arguments.length, typeName, adapter, + 'argAdaptor()', xcv.arg); + }; + + /** + Functions like xCall() but performs argument and result type + conversions as for xWrap(). The first argument is the name of the + exported function to call. The 2nd its the name of its result + type, as documented for xWrap(). The 3rd is an array of argument + type name, as documented for xWrap() (use a falsy value or an + empty array for nullary functions). The 4th+ arguments are + arguments for the call, with the special case that if the 4th + argument is an array, it is used as the arguments for the call + (again, falsy or an empty array for nullary functions). Returns + the converted result of the call. + + This is just a thin wrapp around xWrap(). If the given function + is to be called more than once, it's more efficient to use + xWrap() to create a wrapper, then to call that wrapper as many + times as needed. For one-shot calls, however, this variant is + arguably more efficient because it will hypothetically free the + wrapper function quickly. + */ + target.xCallWrapped = function(fname, resultType, argTypes, ...args){ + if(Array.isArray(arguments[3])) args = arguments[3]; + return this.xWrap(fname, resultType, argTypes||[]).apply(null, args||[]); + }.bind(target); + + return target; +}; + +/** + yawl (Yet Another Wasm Loader) provides very basic wasm loader. + It requires a config object: + + - `uri`: required URI of the WASM file to load. + + - `onload(loadResult,config)`: optional callback. The first + argument is the result object from + WebAssembly.instanitate[Streaming](). The 2nd is the config + object passed to this function. Described in more detail below. + + - `imports`: optional imports object for + WebAssembly.instantiate[Streaming](). The default is am empty set + of imports. If the module requires any imports, this object + must include them. + + - `wasmUtilTarget`: optional object suitable for passing to + WhWasmUtilInstaller(). If set, it gets passed to that function + after the promise resolves. This function sets several properties + on it before passing it on to that function (which sets many + more): + + - `module`, `instance`: the properties from the + instantiate[Streaming]() result. + + - If `instance.exports.memory` is _not_ set then it requires that + `config.imports.env.memory` be set (else it throws), and + assigns that to `target.memory`. + + - If `wasmUtilTarget.alloc` is not set and + `instance.exports.malloc` is, it installs + `wasmUtilTarget.alloc()` and `wasmUtilTarget.dealloc()` + wrappers for the exports `malloc` and `free` functions. + + It returns a function which, when called, initiates loading of the + module and returns a Promise. When that Promise resolves, it calls + the `config.onload` callback (if set) and passes it + `(loadResult,config)`, where `loadResult` is the result of + WebAssembly.instantiate[Streaming](): an object in the form: + + ``` + { + module: a WebAssembly.Module, + instance: a WebAssembly.Instance + } + ``` + + (Note that the initial `then()` attached to the promise gets only + that object, and not the `config` one.) + + Error handling is up to the caller, who may attach a `catch()` call + to the promise. +*/ +self.WhWasmUtilInstaller.yawl = function(config){ + const wfetch = ()=>fetch(config.uri, {credentials: 'same-origin'}); + const wui = this; + const finalThen = function(arg){ + //log("finalThen()",arg); + if(config.wasmUtilTarget){ + const toss = (...args)=>{throw new Error(args.join(' '))}; + const tgt = config.wasmUtilTarget; + tgt.module = arg.module; + tgt.instance = arg.instance; + //tgt.exports = tgt.instance.exports; + if(!tgt.instance.exports.memory){ + /** + WhWasmUtilInstaller requires either tgt.exports.memory + (exported from WASM) or tgt.memory (JS-provided memory + imported into WASM). + */ + tgt.memory = (config.imports && config.imports.env + && config.imports.env.memory) + || toss("Missing 'memory' object!"); + } + if(!tgt.alloc && arg.instance.exports.malloc){ + tgt.alloc = function(n){ + return this(n) || toss("Allocation of",n,"bytes failed."); + }.bind(arg.instance.exports.malloc); + tgt.dealloc = function(m){this(m)}.bind(arg.instance.exports.free); + } + wui(tgt); + } + if(config.onload) config.onload(arg,config); + return arg /* for any then() handler attached to + yetAnotherWasmLoader()'s return value */; + }; + const loadWasm = WebAssembly.instantiateStreaming + ? function loadWasmStreaming(){ + return WebAssembly.instantiateStreaming(wfetch(), config.imports||{}) + .then(finalThen); + } + : function loadWasmOldSchool(){ // Safari < v15 + return wfetch() + .then(response => response.arrayBuffer()) + .then(bytes => WebAssembly.instantiate(bytes, config.imports||{})) + .then(finalThen); + }; + return loadWasm; +}.bind(self.WhWasmUtilInstaller)/*yawl()*/; diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js new file mode 100644 index 0000000000..a018658579 --- /dev/null +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -0,0 +1,737 @@ +/** + 2022-06-30 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + The Jaccwabyt API is documented in detail in an external file. + + Project home: https://fossil.wanderinghorse.net/r/jaccwabyt + +*/ +'use strict'; +self.Jaccwabyt = function StructBinderFactory(config){ +/* ^^^^ it is recommended that clients move that object into wherever + they'd like to have it and delete the self-held copy ("self" being + the global window or worker object). This API does not require the + global reference - it is simply installed as a convenience for + connecting these bits to other co-developed code before it gets + removed from the global namespace. +*/ + + /** Throws a new Error, the message of which is the concatenation + all args with a space between each. */ + const toss = (...args)=>{throw new Error(args.join(' '))}; + + /** + Implementing function bindings revealed significant + shortcomings in Emscripten's addFunction()/removeFunction() + interfaces: + + https://github.com/emscripten-core/emscripten/issues/17323 + + Until those are resolved, or a suitable replacement can be + implemented, our function-binding API will be more limited + and/or clumsier to use than initially hoped. + */ + if(!(config.heap instanceof WebAssembly.Memory) + && !(config.heap instanceof Function)){ + toss("config.heap must be WebAssembly.Memory instance or a function."); + } + ['alloc','dealloc'].forEach(function(k){ + (config[k] instanceof Function) || + toss("Config option '"+k+"' must be a function."); + }); + const SBF = StructBinderFactory; + const heap = (config.heap instanceof Function) + ? config.heap : (()=>new Uint8Array(config.heap.buffer)), + alloc = config.alloc, + dealloc = config.dealloc, + log = config.log || console.log.bind(console), + memberPrefix = (config.memberPrefix || ""), + memberSuffix = (config.memberSuffix || ""), + bigIntEnabled = (undefined===config.bigIntEnabled + ? !!self['BigInt64Array'] : !!config.bigIntEnabled), + BigInt = self['BigInt'], + BigInt64Array = self['BigInt64Array'], + /* Undocumented (on purpose) config options: */ + functionTable = config.functionTable/*EXPERIMENTAL, undocumented*/, + ptrSizeof = config.ptrSizeof || 4, + ptrIR = config.ptrIR || 'i32' + ; + + if(!SBF.debugFlags){ + SBF.__makeDebugFlags = function(deriveFrom=null){ + /* This is disgustingly overengineered. :/ */ + if(deriveFrom && deriveFrom.__flags) deriveFrom = deriveFrom.__flags; + const f = function f(flags){ + if(0===arguments.length){ + return f.__flags; + } + if(flags<0){ + delete f.__flags.getter; delete f.__flags.setter; + delete f.__flags.alloc; delete f.__flags.dealloc; + }else{ + f.__flags.getter = 0!==(0x01 & flags); + f.__flags.setter = 0!==(0x02 & flags); + f.__flags.alloc = 0!==(0x04 & flags); + f.__flags.dealloc = 0!==(0x08 & flags); + } + return f._flags; + }; + Object.defineProperty(f,'__flags', { + iterable: false, writable: false, + value: Object.create(deriveFrom) + }); + if(!deriveFrom) f(0); + return f; + }; + SBF.debugFlags = SBF.__makeDebugFlags(); + }/*static init*/ + + const isLittleEndian = (function() { + const buffer = new ArrayBuffer(2); + new DataView(buffer).setInt16(0, 256, true /* littleEndian */); + // Int16Array uses the platform's endianness. + return new Int16Array(buffer)[0] === 256; + })(); + /** + Some terms used in the internal docs: + + StructType: a struct-wrapping class generated by this + framework. + DEF: struct description object. + SIG: struct member signature string. + */ + + /** True if SIG s looks like a function signature, else + false. */ + const isFuncSig = (s)=>'('===s[1]; + /** True if SIG s is-a pointer signature. */ + const isPtrSig = (s)=>'p'===s || 'P'===s; + const isAutoPtrSig = (s)=>'P'===s /*EXPERIMENTAL*/; + const sigLetter = (s)=>isFuncSig(s) ? 'p' : s[0]; + /** Returns the WASM IR form of the Emscripten-conventional letter + at SIG s[0]. Throws for an unknown SIG. */ + const sigIR = function(s){ + switch(sigLetter(s)){ + case 'i': return 'i32'; + case 'p': case 'P': case 's': return ptrIR; + case 'j': return 'i64'; + case 'f': return 'float'; + case 'd': return 'double'; + } + toss("Unhandled signature IR:",s); + }; + /** Returns the sizeof value for the given SIG. Throws for an + unknown SIG. */ + const sigSizeof = function(s){ + switch(sigLetter(s)){ + case 'i': return 4; + case 'p': case 'P': case 's': return ptrSizeof; + case 'j': return 8; + case 'f': return 4 /* C-side floats, not JS-side */; + case 'd': return 8; + } + toss("Unhandled signature sizeof:",s); + }; + const affirmBigIntArray = BigInt64Array + ? ()=>true : ()=>toss('BigInt64Array is not available.'); + /** Returns the (signed) TypedArray associated with the type + described by the given SIG. Throws for an unknown SIG. */ + /********** + const sigTypedArray = function(s){ + switch(sigIR(s)) { + case 'i32': return Int32Array; + case 'i64': return affirmBigIntArray() && BigInt64Array; + case 'float': return Float32Array; + case 'double': return Float64Array; + } + toss("Unhandled signature TypedArray:",s); + }; + **************/ + /** Returns the name of a DataView getter method corresponding + to the given SIG. */ + const sigDVGetter = function(s){ + switch(sigLetter(s)) { + case 'p': case 'P': case 's': { + switch(ptrSizeof){ + case 4: return 'getInt32'; + case 8: return affirmBigIntArray() && 'getBigInt64'; + } + break; + } + case 'i': return 'getInt32'; + case 'j': return affirmBigIntArray() && 'getBigInt64'; + case 'f': return 'getFloat32'; + case 'd': return 'getFloat64'; + } + toss("Unhandled DataView getter for signature:",s); + }; + /** Returns the name of a DataView setter method corresponding + to the given SIG. */ + const sigDVSetter = function(s){ + switch(sigLetter(s)){ + case 'p': case 'P': case 's': { + switch(ptrSizeof){ + case 4: return 'setInt32'; + case 8: return affirmBigIntArray() && 'setBigInt64'; + } + break; + } + case 'i': return 'setInt32'; + case 'j': return affirmBigIntArray() && 'setBigInt64'; + case 'f': return 'setFloat32'; + case 'd': return 'setFloat64'; + } + toss("Unhandled DataView setter for signature:",s); + }; + /** + Returns either Number of BigInt, depending on the given + SIG. This constructor is used in property setters to coerce + the being-set value to the correct size. + */ + const sigDVSetWrapper = function(s){ + switch(sigLetter(s)) { + case 'i': case 'f': case 'd': return Number; + case 'j': return affirmBigIntArray() && BigInt; + case 'p': case 'P': case 's': + switch(ptrSizeof){ + case 4: return Number; + case 8: return affirmBigIntArray() && BigInt; + } + break; + } + toss("Unhandled DataView set wrapper for signature:",s); + }; + + const sPropName = (s,k)=>s+'::'+k; + + const __propThrowOnSet = function(structName,propName){ + return ()=>toss(sPropName(structName,propName),"is read-only."); + }; + + /** + When C code passes a pointer of a bound struct to back into + a JS function via a function pointer struct member, it + arrives in JS as a number (pointer). + StructType.instanceForPointer(ptr) can be used to get the + instance associated with that pointer, and __ptrBacklinks + holds that mapping. WeakMap keys must be objects, so we + cannot use a weak map to map pointers to instances. We use + the StructType constructor as the WeakMap key, mapped to a + plain, prototype-less Object which maps the pointers to + struct instances. That arrangement gives us a + per-StructType type-safe way to resolve pointers. + */ + const __ptrBacklinks = new WeakMap(); + /** + Similar to __ptrBacklinks but is scoped at the StructBinder + level and holds pointer-to-object mappings for all struct + instances created by any struct from any StructFactory + which this specific StructBinder has created. The intention + of this is to help implement more transparent handling of + pointer-type property resolution. + */ + const __ptrBacklinksGlobal = Object.create(null); + + /** + In order to completely hide StructBinder-bound struct + pointers from JS code, we store them in a scope-local + WeakMap which maps the struct-bound objects to their WASM + pointers. The pointers are accessible via + boundObject.pointer, which is gated behind an accessor + function, but are not exposed anywhere else in the + object. The main intention of that is to make it impossible + for stale copies to be made. + */ + const __instancePointerMap = new WeakMap(); + + /** Property name for the pointer-is-external marker. */ + const xPtrPropName = '(pointer-is-external)'; + + /** Frees the obj.pointer memory and clears the pointer + property. */ + const __freeStruct = function(ctor, obj, m){ + if(!m) m = __instancePointerMap.get(obj); + if(m) { + if(obj.ondispose instanceof Function){ + try{obj.ondispose()} + catch(e){ + /*do not rethrow: destructors must not throw*/ + console.warn("ondispose() for",ctor.structName,'@', + m,'threw. NOT propagating it.',e); + } + }else if(Array.isArray(obj.ondispose)){ + obj.ondispose.forEach(function(x){ + try{ + if(x instanceof Function) x.call(obj); + else if('number' === typeof x) dealloc(x); + // else ignore. Strings are permitted to annotate entries + // to assist in debugging. + }catch(e){ + console.warn("ondispose() for",ctor.structName,'@', + m,'threw. NOT propagating it.',e); + } + }); + } + delete obj.ondispose; + delete __ptrBacklinks.get(ctor)[m]; + delete __ptrBacklinksGlobal[m]; + __instancePointerMap.delete(obj); + if(ctor.debugFlags.__flags.dealloc){ + log("debug.dealloc:",(obj[xPtrPropName]?"EXTERNAL":""), + ctor.structName,"instance:", + ctor.structInfo.sizeof,"bytes @"+m); + } + if(!obj[xPtrPropName]) dealloc(m); + } + }; + + /** Returns a skeleton for a read-only property accessor wrapping + value v. */ + const rop = (v)=>{return {configurable: false, writable: false, + iterable: false, value: v}}; + + /** Allocates obj's memory buffer based on the size defined in + DEF.sizeof. */ + const __allocStruct = function(ctor, obj, m){ + let fill = !m; + if(m) Object.defineProperty(obj, xPtrPropName, rop(m)); + else{ + m = alloc(ctor.structInfo.sizeof); + if(!m) toss("Allocation of",ctor.structName,"structure failed."); + } + try { + if(ctor.debugFlags.__flags.alloc){ + log("debug.alloc:",(fill?"":"EXTERNAL"), + ctor.structName,"instance:", + ctor.structInfo.sizeof,"bytes @"+m); + } + if(fill) heap().fill(0, m, m + ctor.structInfo.sizeof); + __instancePointerMap.set(obj, m); + __ptrBacklinks.get(ctor)[m] = obj; + __ptrBacklinksGlobal[m] = obj; + }catch(e){ + __freeStruct(ctor, obj, m); + throw e; + } + }; + /** Gets installed as the memoryDump() method of all structs. */ + const __memoryDump = function(){ + const p = this.pointer; + return p + ? new Uint8Array(heap().slice(p, p+this.structInfo.sizeof)) + : null; + }; + + const __memberKey = (k)=>memberPrefix + k + memberSuffix; + const __memberKeyProp = rop(__memberKey); + + /** + Looks up a struct member in structInfo.members. Throws if found + if tossIfNotFound is true, else returns undefined if not + found. The given name may be either the name of the + structInfo.members key (faster) or the key as modified by the + memberPrefix/memberSuffix settings. + */ + const __lookupMember = function(structInfo, memberName, tossIfNotFound=true){ + let m = structInfo.members[memberName]; + if(!m && (memberPrefix || memberSuffix)){ + // Check for a match on members[X].key + for(const v of Object.values(structInfo.members)){ + if(v.key===memberName){ m = v; break; } + } + if(!m && tossIfNotFound){ + toss(sPropName(structInfo.name,memberName),'is not a mapped struct member.'); + } + } + return m; + }; + + /** + Uses __lookupMember(obj.structInfo,memberName) to find a member, + throwing if not found. Returns its signature, either in this + framework's native format or in Emscripten format. + */ + const __memberSignature = function f(obj,memberName,emscriptenFormat=false){ + if(!f._) f._ = (x)=>x.replace(/[^vipPsjrd]/g,'').replace(/[pPs]/g,'i'); + const m = __lookupMember(obj.structInfo, memberName, true); + return emscriptenFormat ? f._(m.signature) : m.signature; + }; + + /** + Returns the instanceForPointer() impl for the given + StructType constructor. + */ + const __instanceBacklinkFactory = function(ctor){ + const b = Object.create(null); + __ptrBacklinks.set(ctor, b); + return (ptr)=>b[ptr]; + }; + + const __ptrPropDescriptor = { + configurable: false, enumerable: false, + get: function(){return __instancePointerMap.get(this)}, + set: ()=>toss("Cannot assign the 'pointer' property of a struct.") + // Reminder: leaving `set` undefined makes assignments + // to the property _silently_ do nothing. Current unit tests + // rely on it throwing, though. + }; + + /** Impl of X.memberKeys() for StructType and struct ctors. */ + const __structMemberKeys = rop(function(){ + const a = []; + Object.keys(this.structInfo.members).forEach((k)=>a.push(this.memberKey(k))); + return a; + }); + + const __utf8Decoder = new TextDecoder('utf-8'); + const __utf8Encoder = new TextEncoder(); + + /** + Uses __lookupMember() to find the given obj.structInfo key. + Returns that member if it is a string, else returns false. If the + member is not found, throws if tossIfNotFound is true, else + returns false. + */ + const __memberIsString = function(obj,memberName, tossIfNotFound=false){ + const m = __lookupMember(obj.structInfo, memberName, tossIfNotFound); + return (m && 1===m.signature.length && 's'===m.signature[0]) ? m : false; + }; + + /** + Given a member description object, throws if member.signature is + not valid for assigning to or interpretation as a C-style string. + It optimistically assumes that any signature of (i,p,s) is + C-string compatible. + */ + const __affirmCStringSignature = function(member){ + if('s'===member.signature) return; + toss("Invalid member type signature for C-string value:", + JSON.stringify(member)); + }; + + /** + Looks up the given member in obj.structInfo. If it has a + signature of 's' then it is assumed to be a C-style UTF-8 string + and a decoded copy of the string at its address is returned. If + the signature is of any other type, it throws. If an s-type + member's address is 0, `null` is returned. + */ + const __memberToJsString = function f(obj,memberName){ + const m = __lookupMember(obj.structInfo, memberName, true); + __affirmCStringSignature(m); + const addr = obj[m.key]; + //log("addr =",addr,memberName,"m =",m); + if(!addr) return null; + let pos = addr; + const mem = heap(); + for( ; mem[pos]!==0; ++pos ) { + //log("mem[",pos,"]",mem[pos]); + }; + //log("addr =",addr,"pos =",pos); + if(addr===pos) return ""; + return __utf8Decoder.decode(new Uint8Array(mem.buffer, addr, pos-addr)); + }; + + /** + Adds value v to obj.ondispose, creating ondispose, + or converting it to an array, if needed. + */ + const __addOnDispose = function(obj, v){ + if(obj.ondispose){ + if(obj.ondispose instanceof Function){ + obj.ondispose = [obj.ondispose]; + }/*else assume it's an array*/ + }else{ + obj.ondispose = []; + } + obj.ondispose.push(v); + }; + + /** + Allocates a new UTF-8-encoded, NUL-terminated copy of the given + JS string and returns its address relative to heap(). If + allocation returns 0 this function throws. Ownership of the + memory is transfered to the caller, who must eventually pass it + to the configured dealloc() function. + */ + const __allocCString = function(str){ + const u = __utf8Encoder.encode(str); + const mem = alloc(u.length+1); + if(!mem) toss("Allocation error while duplicating string:",str); + const h = heap(); + let i = 0; + for( ; i < u.length; ++i ) h[mem + i] = u[i]; + h[mem + u.length] = 0; + //log("allocCString @",mem," =",u); + return mem; + }; + + /** + Sets the given struct member of obj to a dynamically-allocated, + UTF-8-encoded, NUL-terminated copy of str. It is up to the caller + to free any prior memory, if appropriate. The newly-allocated + string is added to obj.ondispose so will be freed when the object + is disposed. + */ + const __setMemberCString = function(obj, memberName, str){ + const m = __lookupMember(obj.structInfo, memberName, true); + __affirmCStringSignature(m); + /* Potential TODO: if obj.ondispose contains obj[m.key] then + dealloc that value and clear that ondispose entry */ + const mem = __allocCString(str); + obj[m.key] = mem; + __addOnDispose(obj, mem); + return obj; + }; + + /** + Prototype for all StructFactory instances (the constructors + returned from StructBinder). + */ + const StructType = function ctor(structName, structInfo){ + if(arguments[2]!==rop){ + toss("Do not call the StructType constructor", + "from client-level code."); + } + Object.defineProperties(this,{ + //isA: rop((v)=>v instanceof ctor), + structName: rop(structName), + structInfo: rop(structInfo) + }); + }; + + /** + Properties inherited by struct-type-specific StructType instances + and (indirectly) concrete struct-type instances. + */ + StructType.prototype = Object.create(null, { + dispose: rop(function(){__freeStruct(this.constructor, this)}), + lookupMember: rop(function(memberName, tossIfNotFound=true){ + return __lookupMember(this.structInfo, memberName, tossIfNotFound); + }), + memberToJsString: rop(function(memberName){ + return __memberToJsString(this, memberName); + }), + memberIsString: rop(function(memberName, tossIfNotFound=true){ + return __memberIsString(this, memberName, tossIfNotFound); + }), + memberKey: __memberKeyProp, + memberKeys: __structMemberKeys, + memberSignature: rop(function(memberName, emscriptenFormat=false){ + return __memberSignature(this, memberName, emscriptenFormat); + }), + memoryDump: rop(__memoryDump), + pointer: __ptrPropDescriptor, + setMemberCString: rop(function(memberName, str){ + return __setMemberCString(this, memberName, str); + }) + }); + + /** + "Static" properties for StructType. + */ + Object.defineProperties(StructType, { + allocCString: rop(__allocCString), + instanceForPointer: rop((ptr)=>__ptrBacklinksGlobal[ptr]), + isA: rop((v)=>v instanceof StructType), + hasExternalPointer: rop((v)=>(v instanceof StructType) && !!v[xPtrPropName]), + memberKey: __memberKeyProp + }); + + const isNumericValue = (v)=>Number.isFinite(v) || (v instanceof (BigInt || Number)); + + /** + Pass this a StructBinder-generated prototype, and the struct + member description object. It will define property accessors for + proto[memberKey] which read from/write to memory in + this.pointer. It modifies descr to make certain downstream + operations much simpler. + */ + const makeMemberWrapper = function f(ctor,name, descr){ + if(!f._){ + /*cache all available getters/setters/set-wrappers for + direct reuse in each accessor function. */ + f._ = {getters: {}, setters: {}, sw:{}}; + const a = ['i','p','P','s','f','d','v()']; + if(bigIntEnabled) a.push('j'); + a.forEach(function(v){ + //const ir = sigIR(v); + f._.getters[v] = sigDVGetter(v) /* DataView[MethodName] values for GETTERS */; + f._.setters[v] = sigDVSetter(v) /* DataView[MethodName] values for SETTERS */; + f._.sw[v] = sigDVSetWrapper(v) /* BigInt or Number ctor to wrap around values + for conversion */; + }); + const rxSig1 = /^[ipPsjfd]$/, + rxSig2 = /^[vipPsjfd]\([ipPsjfd]*\)$/; + f.sigCheck = function(obj, name, key,sig){ + if(Object.prototype.hasOwnProperty.call(obj, key)){ + toss(obj.structName,'already has a property named',key+'.'); + } + rxSig1.test(sig) || rxSig2.test(sig) + || toss("Malformed signature for", + sPropName(obj.structName,name)+":",sig); + }; + } + const key = ctor.memberKey(name); + f.sigCheck(ctor.prototype, name, key, descr.signature); + descr.key = key; + descr.name = name; + const sizeOf = sigSizeof(descr.signature); + const sigGlyph = sigLetter(descr.signature); + const xPropName = sPropName(ctor.prototype.structName,key); + const dbg = ctor.prototype.debugFlags.__flags; + /* + TODO?: set prototype of descr to an object which can set/fetch + its prefered representation, e.g. conversion to string or mapped + function. Advantage: we can avoid doing that via if/else if/else + in the get/set methods. + */ + const prop = Object.create(null); + prop.configurable = false; + prop.enumerable = false; + prop.get = function(){ + if(dbg.getter){ + log("debug.getter:",f._.getters[sigGlyph],"for", sigIR(sigGlyph), + xPropName,'@', this.pointer,'+',descr.offset,'sz',sizeOf); + } + let rc = ( + new DataView(heap().buffer, this.pointer + descr.offset, sizeOf) + )[f._.getters[sigGlyph]](0, isLittleEndian); + if(dbg.getter) log("debug.getter:",xPropName,"result =",rc); + if(rc && isAutoPtrSig(descr.signature)){ + rc = StructType.instanceForPointer(rc) || rc; + if(dbg.getter) log("debug.getter:",xPropName,"resolved =",rc); + } + return rc; + }; + if(descr.readOnly){ + prop.set = __propThrowOnSet(ctor.prototype.structName,key); + }else{ + prop.set = function(v){ + if(dbg.setter){ + log("debug.setter:",f._.setters[sigGlyph],"for", sigIR(sigGlyph), + xPropName,'@', this.pointer,'+',descr.offset,'sz',sizeOf, v); + } + if(!this.pointer){ + toss("Cannot set struct property on disposed instance."); + } + if(null===v) v = 0; + else while(!isNumericValue(v)){ + if(isAutoPtrSig(descr.signature) && (v instanceof StructType)){ + // It's a struct instance: let's store its pointer value! + v = v.pointer || 0; + if(dbg.setter) log("debug.setter:",xPropName,"resolved to",v); + break; + } + toss("Invalid value for pointer-type",xPropName+'.'); + } + ( + new DataView(heap().buffer, this.pointer + descr.offset, sizeOf) + )[f._.setters[sigGlyph]](0, f._.sw[sigGlyph](v), isLittleEndian); + }; + } + Object.defineProperty(ctor.prototype, key, prop); + }/*makeMemberWrapper*/; + + /** + The main factory function which will be returned to the + caller. + */ + const StructBinder = function StructBinder(structName, structInfo){ + if(1===arguments.length){ + structInfo = structName; + structName = structInfo.name; + }else if(!structInfo.name){ + structInfo.name = structName; + } + if(!structName) toss("Struct name is required."); + let lastMember = false; + Object.keys(structInfo.members).forEach((k)=>{ + const m = structInfo.members[k]; + if(!m.sizeof) toss(structName,"member",k,"is missing sizeof."); + else if(0!==(m.sizeof%4)){ + toss(structName,"member",k,"sizeof is not aligned."); + } + else if(0!==(m.offset%4)){ + toss(structName,"member",k,"offset is not aligned."); + } + if(!lastMember || lastMember.offset < m.offset) lastMember = m; + }); + if(!lastMember) toss("No member property descriptions found."); + else if(structInfo.sizeof < lastMember.offset+lastMember.sizeof){ + toss("Invalid struct config:",structName, + "max member offset ("+lastMember.offset+") ", + "extends past end of struct (sizeof="+structInfo.sizeof+")."); + } + const debugFlags = rop(SBF.__makeDebugFlags(StructBinder.debugFlags)); + /** Constructor for the StructCtor. */ + const StructCtor = function StructCtor(externalMemory){ + if(!(this instanceof StructCtor)){ + toss("The",structName,"constructor may only be called via 'new'."); + }else if(arguments.length){ + if(externalMemory!==(externalMemory|0) || externalMemory<=0){ + toss("Invalid pointer value for",structName,"constructor."); + } + __allocStruct(StructCtor, this, externalMemory); + }else{ + __allocStruct(StructCtor, this); + } + }; + Object.defineProperties(StructCtor,{ + debugFlags: debugFlags, + disposeAll: rop(function(){ + const map = __ptrBacklinks.get(StructCtor); + Object.keys(map).forEach(function(ptr){ + const b = map[ptr]; + if(b) __freeStruct(StructCtor, b, ptr); + }); + __ptrBacklinks.set(StructCtor, Object.create(null)); + return StructCtor; + }), + instanceForPointer: rop(__instanceBacklinkFactory(StructCtor)), + isA: rop((v)=>v instanceof StructCtor), + memberKey: __memberKeyProp, + memberKeys: __structMemberKeys, + resolveToInstance: rop(function(v, throwIfNot=false){ + if(!(v instanceof StructCtor)){ + v = Number.isSafeInteger(v) + ? StructCtor.instanceForPointer(v) : undefined; + } + if(!v && throwIfNot) toss("Value is-not-a",StructCtor.structName); + return v; + }), + methodInfoForKey: rop(function(mKey){ + }), + structInfo: rop(structInfo), + structName: rop(structName) + }); + StructCtor.prototype = new StructType(structName, structInfo, rop); + Object.defineProperties(StructCtor.prototype,{ + debugFlags: debugFlags, + constructor: rop(StructCtor) + /*if we assign StructCtor.prototype and don't do + this then StructCtor!==instance.constructor!*/ + }); + Object.keys(structInfo.members).forEach( + (name)=>makeMemberWrapper(StructCtor, name, structInfo.members[name]) + ); + return StructCtor; + }; + StructBinder.instanceForPointer = StructType.instanceForPointer; + StructBinder.StructType = StructType; + StructBinder.config = config; + StructBinder.allocCString = __allocCString; + if(!StructBinder.debugFlags){ + StructBinder.debugFlags = SBF.__makeDebugFlags(SBF.debugFlags); + } + return StructBinder; +}/*StructBinderFactory*/; diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md new file mode 100644 index 0000000000..2bb39e636f --- /dev/null +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -0,0 +1,1078 @@ +Jaccwabyt 🐇 +============================================================ + +**Jaccwabyt**: _JavaScript ⇄ C Struct Communication via WASM Byte +Arrays_ + + +Welcome to Jaccwabyt, a JavaScript API which creates bindings for +WASM-compiled C structs, defining them in such a way that changes to +their state in JS are visible in C/WASM, and vice versa, permitting +two-way interchange of struct state with very little user-side +friction. + +(If that means nothing to you, neither will the rest of this page!) + +**Browser compatibility**: this library requires a _recent_ browser +and makes no attempt whatsoever to accommodate "older" or +lesser-capable ones, where "recent," _very roughly_, means released in +mid-2018 or later, with late 2021 releases required for some optional +features in some browsers (e.g. [BigInt64Array][] in Safari). It also +relies on a couple non-standard, but widespread, features, namely +[TextEncoder][] and [TextDecoder][]. It is developed primarily on +Firefox and Chrome on Linux and all claims of Safari compatibility +are based solely on feature compatibility tables provided at +[MDN][]. + +**Formalities:** + +- Author: [Stephan Beal][sgb] +- License: Public Domain +- Project Home: + + +Table of Contents +============================================================ + +- [Overview](#overview) + - [Architecture](#architecture) +- [Creating and Binding Structs](#creating-binding) + - [Step 1: Configure Jaccwabyt](#step-1) + - [Step 2: Struct Description](#step-2) + - [`P` vs `p`](#step-2-pvsp) + - [Step 3: Binding a Struct](#step-3) + - [Step 4: Creating, Using, and Destroying Instances](#step-4) +- APIs + - [Struct Binder Factory](#api-binderfactory) + - [Struct Binder](#api-structbinder) + - [Struct Type](#api-structtype) + - [Struct Constructors](#api-structctor) + - [Struct Protypes](#api-structprototype) + - [Struct Instances](#api-structinstance) +- Appendices + - [Appendix A: Limitations, TODOs, etc.](#appendix-a) + - [Appendix D: Debug Info](#appendix-d) + - [Appendix G: Generating Struct Descriptions](#appendix-g) + + +Overview +============================================================ + +Management summary: this JavaScript-only framework provides limited +two-way bindings between C structs and JavaScript objects, such that +changes to the struct in one environment are visible in the other. + +Details... + +It works by creating JavaScript proxies for C structs. Reads and +writes of the JS-side members are marshaled through a flat byte array +allocated from the WASM heap. As that heap is shared with the C-side +code, and the memory block is written using the same approach C does, +that byte array can be used to access and manipulate a given struct +instance from both JS and C. + +Motivating use case: this API was initially developed as an +experiment to determine whether it would be feasible to implement, +completely in JS, custom "VFS" and "virtual table" objects for the +WASM build of [sqlite3][]. Doing so was going to require some form of +two-way binding of several structs. Once the proof of concept was +demonstrated, a rabbit hole appeared and _down we went_... It has +since grown beyond its humble proof-of-concept origins and is believed +to be a useful (or at least interesting) tool for mixed JS/C +applications. + +Portability notes: + +- These docs sometimes use [Emscripten][] as a point of reference + because it is the most widespread WASM toolchain, but this code is + specifically designed to be usable in arbitrary WASM environments. + It abstracts away a few Emscripten-specific features into + configurable options. Similarly, the build tree requires Emscripten + but Jaccwabyt does not have any hard Emscripten dependencies. +- This code is encapsulated into a single JavaScript function. It + should be trivial to copy/paste into arbitrary WASM/JS-using + projects. +- The source tree includes C code, but only for testing and + demonstration purposes. It is not part of the core distributable. + + +Architecture +------------------------------------------------------------ + + + +```pikchr +BSBF: box rad 0.3*boxht "StructBinderFactory" fit fill lightblue +BSB: box same "StructBinder" fit at 0.75 e of 0.7 s of BSBF.c +BST: box same "StructType" fit at 1.5 e of BSBF +BSC: box same "Struct" "Ctor" fit at 1.5 s of BST +BSI: box same "Struct" "Instances" fit at 1 right of BSB.e +BC: box same at 0.25 right of 1.6 e of BST "C Structs" fit fill lightgrey + +arrow -> from BSBF.s to BSB.w "Generates" aligned above +arrow -> from BSB.n to BST.sw "Contains" aligned above +arrow -> from BSB.s to BSC.nw "Generates" aligned below +arrow -> from BSC.ne to BSI.s "Constructs" aligned below +arrow <- from BST.se to BSI.n "Inherits" aligned above +arrow <-> from BSI.e to BC.s dotted "Shared" aligned above "Memory" aligned below +arrow -> from BST.e to BC.w dotted "Mirrors Struct" aligned above "Model From" aligned below +arrow -> from BST.s to BSC.n "Prototype of" aligned above +``` + +Its major classes and functions are: + +- **[StructBinderFactory][StructBinderFactory]** is a factory function which + accepts a configuration object to customize it for a given WASM + environment. A client will typically call this only one time, with + an appropriate configuration, to generate a single... +- **[StructBinder][]** is a factory function which converts an + arbitrary number struct descriptions into... +- **[StructTypes][StructCtors]** are constructors, one per struct + description, which inherit from + **[`StructBinder.StructType`][StructType]** and are used to instantiate... +- **[Struct instances][StructInstance]** are objects representing + individual instances of generated struct types. + +An app may have any number of StructBinders, but will typically +need only one. Each StructBinder is effectively a separate +namespace for struct creation. + + + +Creating and Binding Structs +============================================================ + +From the amount of documentation provided, it may seem that +creating and using struct bindings is a daunting task, but it +essentially boils down to: + +1. [Confire Jaccwabyt for your WASM environment](#step-1). This is a + one-time task per project and results is a factory function which + can create new struct bindings. +2. [Create a JSON-format description of your C structs](#step-2). This is + required once for each struct and required updating if the C + structs change. +3. [Feed (2) to the function generated by (1)](#step-3) to create JS + constuctor functions for each struct. This is done at runtime, as + opposed to during a build-process step, and can be set up in such a + way that it does not require any maintenace after its initial + setup. +4. [Create and use instances of those structs](#step-4). + +Detailed instructions for each of those steps follows... + + +Step 1: Configure Jaccwabyt for the Environment +------------------------------------------------------------ + +Jaccwabyt's highest-level API is a single function. It creates a +factory for processing struct descriptions, but does not process any +descriptions itself. This level of abstraction exist primarily so that +the struct-specific factories can be configured for a given WASM +environment. Its usage looks like: + +> +```javascript +const MyBinder = StructBinderFactory({ + // These config options are all required: + heap: WebAssembly.Memory instance or a function which returns + a Uint8Array or Int8Array view of the WASM memory, + alloc: function(howMuchMemory){...}, + dealloc: function(pointerToFree){...} +}); +``` + +It also offers a number of other settings, but all are optional except +for the ones shown above. Those three config options abstract away +details which are specific to a given WASM environment. They provide +the WASM "heap" memory (a byte array), the memory allocator, and the +deallocator. In a conventional Emscripten setup, that config might +simply look like: + +> +```javascript +{ + heap: Module['asm']['memory'], + //Or: + // heap: ()=>Module['HEAP8'], + alloc: (n)=>Module['_malloc'](n), + dealloc: (m)=>Module['_free'](m) +} +``` + +The StructBinder factory function returns a function which can then be +used to create bindings for our structs. + + + +Step 2: Create a Struct Description +------------------------------------------------------------ + +The primary input for this framework is a JSON-compatible construct +which describes a struct we want to bind. For example, given this C +struct: + +> +```c +// C-side: +struct Foo { + int member1; + void * member2; + int64_t member3; +}; +``` + +Its JSON description looks like: + +> +```json +{ + "name": "Foo", + "sizeof": 16, + "members": { + "member1": {"offset": 0,"sizeof": 4,"signature": "i"}, + "member2": {"offset": 4,"sizeof": 4,"signature": "p"}, + "member3": {"offset": 8,"sizeof": 8,"signature": "j"} + } +} +``` + +These data _must_ match up with the C-side definition of the struct +(if any). See [Appendix G][appendix-g] for one way to easily generate +these from C code. + +Each entry in the `members` object maps the member's name to +its low-level layout: + +- `offset`: the byte offset from the start of the struct, as reported + by C's `offsetof()` feature. +- `sizeof`: as reported by C's `sizeof()`. +- `signature`: described below. +- `readOnly`: optional. If set to true, the binding layer will + throw if JS code tries to set that property. + +The order of the `members` entries is not important: their memory +layout is determined by their `offset` and `sizeof` members. The +`name` property is technically optional, but one of the steps in the +binding process requires that either it be passed an explicit name or +there be one in the struct description. The names of the `members` +entries need not match their C counterparts. Project conventions may +call for giving them different names in the JS side and the +[StructBinderFactory][] can be configured to automatically add a +prefix and/or suffix to their names. + +Nested structs are as-yet unsupported by this tool. + +Struct member "signatures" describe the data types of the members and +are an extended variant of the format used by Emscripten's +`addFunction()`. A signature for a non-function-pointer member, or +function pointer member which is to be modelled as an opaque pointer, +is a single letter. A signature for a function pointer may also be +modelled as a series of letters describing the call signature. The +supported letters are: + +- **`v`** = `void` (only used as return type for function pointer members) +- **`i`** = `int32` (4 bytes) +- **`j`** = `int64` (8 bytes) is only really usable if this code is built + with BigInt support (e.g. using the Emscripten `-sWASM_BIGINT` build + flag). Without that, this API may throw when encountering the `j` + signature entry. +- **`f`** = `float` (4 bytes) +- **`d`** = `double` (8 bytes) +- **`p`** = `int32` (but see below!) +- **`P`** = Like `p` but with extra handling. Described below. +- **`s`** = like `int32` but is a _hint_ that it's a pointer to a string + so that _some_ (very limited) contexts may treat it as such, noting + such algorithms must, for lack of information to the contrary, + assume both that the encoding is UTF-8 and that the pointer's member + is NUL-terminated. If that is _not_ the case for a given string + member, do not use `s`: use `i` or `p` instead and do any string + handling yourself. + +Noting that: + +- All of these types are numeric. Attempting to set any struct-bound + property to a non-numeric value will trigger an exception except in + cases explicitly noted otherwise. + +> Sidebar: Emscripten's public docs do not mention `p`, but their +generated code includes `p` as an alias for `i`, presumably to mean +"pointer". Though `i` is legal for pointer types in the signature, `p` +is more descriptive, so this framework encourages the use of `p` for +pointer-type members. Using `p` for pointers also helps future-proof +the signatures against the eventuality that WASM eventually supports +64-bit pointers. Note that sometimes `p` really means +pointer-to-pointer, but the Emscripten JS/WASM glue does not offer +that level of expressiveness in these signatures. We simply have to be +aware of when we need to deal with pointers and pointers-to-pointers +in JS code. + +> Trivia: this API treates `p` as distinctly different from `i` in +some contexts, so its use is encouraged for pointer types. + +Signatures in the form `x(...)` denote function-pointer members and +`x` denotes non-function members. Functions with no arguments use the +form `x()`. For function-type signatures, the strings are formulated +such that they can be passed to Emscripten's `addFunction()` after +stripping out the `(` and `)` characters. For good measure, to match +the public Emscripten docs, `p` should also be replaced with `i`. In +JavaScript that might look like: + +> +``` +signature.replace(/[^vipPsjfd]/g,'').replace(/[pPs]/g,'i'); +``` + + +### `P` vs `p` in Method Signatures + +*This support is experimental and subject to change.* + +The method signature letter `p` means "pointer," which, in WASM, means +"integer." `p` is treated as an integer for most contexts, while still +also being a separate type (analog to how pointers in C are just a +special use of unsigned numbers). A capital `P` changes the semantics +of plain member pointers (but not, as of this writing, function +pointer members) as follows: + +- When a `P`-type member is **fetched** via `myStruct.x` and its value is + a non-0 integer, [`StructBinder.instanceForPointer()`][StructBinder] + is used to try to map that pointer to a struct instance. If a match + is found, the "get" operation returns that instance instead of the + integer. If no match is found, it behaves exactly as for `p`, returning + the integer value. +- When a `P`-type member is **set** via `myStruct.x=y`, if + [`(y instanceof StructType)`][StructType] then the value of `y.pointer` is + stored in `myStruct.x`. If `y` is neither a number nor + a [StructType][], an exception is triggered (regardless of whether + `p` or `P` is used). + + + +Step 3: Binding the Struct +------------------------------------------------------------ + +We can now use the results of steps 1 and 2: + +> +```javascript +const MyStruct = MyBinder(myStructDescription); +``` + +That creates a new constructor function, `MyStruct`, which can be used +to instantiate new instances. The binder will throw if it encounters +any problems. + +That's all there is to it. + +> Sidebar: that function may modify the struct description object +and/or its sub-objects, or may even replace sub-objects, in order to +simplify certain later operations. If that is not desired, then feed +it a copy of the original, e.g. by passing it +`JSON.parse(JSON.stringify(structDefinition))`. + + +Step 4: Creating, Using, and Destroying Struct Instances +------------------------------------------------------------ + +Now that we have our constructor... + +> +```javascript +const my = new MyStruct(); +``` + +It is important to understand that creating a new instance allocates +memory on the WASM heap. We must not simply rely on garbage collection +to clean up the instances because doing so will not free up the WASM +heap memory. The correct way to free up that memory is to use the +object's `dispose()` method. Alternately, there is a "nuclear option": +`MyBinder.disposeAll()` will free the memory allocated for _all_ +instances which have not been manually disposed. + +The following usage pattern offers one way to easily ensure proper +cleanup of struct instances: + + +> +```javascript +const my = new MyStruct(); +try { + console.log(my.member1, my.member2, my.member3); + my.member1 = 12; + assert(12 === my.member1); + /* ^^^ it may seem silly to test that, but recall that assigning that + property encodes the value into a byte array in heap memory, not + a normal JS property. Similarly, fetching the property decodes it + from the byte array. */ + // Pass the struct to C code which takes a MyStruct pointer: + aCFunction( my.pointer ); + // Type-safely check if a pointer returned from C is a MyStruct: + const x = MyStruct.instanceForPointer( anotherCFunction() ); + // If it is a MyStruct, x now refers to that object. Note, however, + // that this only works for instances created in JS, as the + // pointer mapping only exists in JS space. +} finally { + my.dispose(); +} +``` + +> Sidebar: the `finally` block will be run no matter how the `try` +exits, whether it runs to completion, propagates an exception, or uses +flow-control keywords like `return` or `break`. It is perfectly legal +to use `try`/`finally` without a `catch`, and doing so is an ideal +match for the memory management requirements of Jaccwaby-bound struct +instances. + +Now that we have struct instances, there are a number of things we +can do with them, as covered in the rest of this document. + + + +API Reference +============================================================ + + +API: Binder Factory +------------------------------------------------------------ + +This is the top-most function of the API, from which all other +functions and types are generated. The binder factory's signature is: + +> +``` +Function StructBinderFactory(object configOptions); +``` + +It returns a function which these docs refer to as a [StructBinder][] +(covered in the next section). It throws on error. + +The binder factory supports the following options in its +configuration object argument: + + +- `heap` + Must be either a `WebAssembly.Memory` instance representing the WASM + heap memory OR a function which returns an Int8Array or Uint8Array + view of the WASM heap. In the latter case the function should, if + appropriate for the environment, account for the heap being able to + grow. Jaccwabyt uses this property in such a way that it "should" be + okay for the WASM heap to grow at runtime (that case is, however, + untested). + +- `alloc` + Must be a function semantically compatible with Emscripten's + `Module._malloc()`. That is, it is passed the number of bytes to + allocate and it returns a pointer. On allocation failure it may + either return 0 or throw an exception. This API will throw an + exception if allocation fails or will propagate whatever exception + the allocator throws. The allocator _must_ use the same heap as the + `heap` config option. + +- `dealloc` + Must be a function semantically compatible with Emscripten's + `Module._free()`. That is, it takes a pointer returned from + `alloc()` and releases that memory. It must never throw and must + accept a value of 0/null to mean "do nothing" (noting that 0 is + _technically_ a legal memory address in WASM, but that seems like a + design flaw). + +- `bigIntEnabled` (bool=true if BigInt64Array is available, else false) + If true, the WASM bits this code is used with must have been + compiled with int64 support (e.g. using Emscripten's `-sWASM_BIGINT` + flag). If that's not the case, this flag should be set to false. If + it's enabled, BigInt support is assumed to work and certain extra + features are enabled. Trying to use features which requires BigInt + when it is disabled (e.g. using 64-bit integer types) will trigger + an exception. + +- `memberPrefix` and `memberSuffix` (string="") + If set, struct-defined properties get bound to JS with this string + as a prefix resp. suffix. This can be used to avoid symbol name + collisions between the struct-side members and the JS-side ones + and/or to make more explicit which object-level properties belong to + the struct mapping and which to the JS side. This does not modify + the values in the struct description objects, just the property + names through which they are accessed via property access operations + and the various a [StructInstance][] APIs (noting that the latter + tend to permit both the original names and the names as modified by + these settings). + +- `log` + Optional function used for debugging output. By default + `console.log` is used but by default no debug output is generated. + This API assumes that the function will space-separate each argument + (like `console.log` does). See [Appendix D](#appendix-d) for info + about enabling debugging output. + + + +API: Struct Binder +------------------------------------------------------------ + +Struct Binders are factories which are created by the +[StructBinderFactory][]. A given Struct Binder can process any number +of distinct structs. In a typical setup, an app will have ony one +shared Binder Factory and one Struct Binder. Struct Binders which are +created via different [StructBinderFactory][] calls are unrelated to each +other, sharing no state except, perhaps, indirectly via +[StructBinderFactory][] configuration (e.g. the memory heap). + +These factories have two call signatures: + +> +```javascript +Function StructBinder([string structName,] object structDescription) +``` + +If the struct description argument has a `name` property then the name +argument is optional, otherwise it is required. + +The returned object is a constructor for instances of the struct +described by its argument(s), each of which derives from +a separate [StructType][] instance. + +The Struct Binder has the following members: + +- `allocCString(str)` + Allocates a new UTF-8-encoded, NUL-terminated copy of the given JS + string and returns its address relative to `config.heap()`. If + allocation returns 0 this function throws. Ownership of the memory + is transfered to the caller, who must eventually pass it to the + configured `config.dealloc()` function. + +- `config` + The configuration object passed to the [StructBinderFactory][], + primarily for accessing the memory (de)allocator and memory. Modifying + any of its "significant" configuration values may have undefined + results. + +- `instanceForPointer(pointer)` + Given a pointer value relative to `config.memory`, if that pointer + resolves to a struct of _any type_ generated via the same Struct + Binder, this returns the struct instance associated with it, or + `undefined` if no struct object is mapped to that pointer. This + differs from the struct-type-specific member of the same name in + that this one is not "type-safe": it does not know the type of the + returned object (if any) and may return a struct of any + [StructType][] for which this Struct Binder has created a + constructor. It cannot return instances created via a different + [StructBinderFactory][] because each factory can hypothetically have + a different memory heap. + + + +API: Struct Type +------------------------------------------------------------ + +The StructType class is a property of the [StructBinder][] function. + +Each constructor created by a [StructBinder][] inherits from _its own +instance_ of the StructType class, which contains state specific to +that struct type (e.g. the struct name and description metadata). +StructTypes which are created via different [StructBinder][] instances +are unrelated to each other, sharing no state except [StructBinderFactory][] +config options. + +The StructType constructor cannot be called from client code. It is +only called by the [StructBinder][]-generated +[constructors][StructCtors]. The `StructBinder.StructType` object +has the following "static" properties (^Which are accessible from +individual instances via `theInstance.constructor`.): + +- `allocCString(str)` + Identical to the [StructBinder][] method of the same name. + +- `hasExternalPointer(object)` + Returns true if the given object's `pointer` member refers to an + "external" object. That is the case when a pointer is passed to a + [struct's constructor][StructCtors]. If true, the memory is owned by + someone other than the object and must outlive the object. + +- `instanceForPointer(pointer)` + Works identically to the [StructBinder][] method of the same name. + +- `isA(value)` + Returns true if its argument is a StructType instance _from the same + [StructBinder][]_ as this StructType. + +- `memberKey(string)` + Returns the given string wrapped in the configured `memberPrefix` + and `memberSuffix` values. e.g. if passed `"x"` and `memberPrefix` + is `"$"` then it returns `"$x"`. This does not verify that the + property is actually a struct a member, it simply transforms the + given string. TODO(?): add a 2nd parameter indicating whether it + should validate that it's a known member name. + +The base StructType prototype has the following members, all of which +are inherited by [struct instances](#api-structinstance) and may only +legally be called on concrete struct instances unless noted otherwise: + +- `dispose()` + Frees, if appropriate, the WASM-allocated memory which is allocated + by the constructor. If this is not called before the JS engine + cleans up the object, a leak in the WASM heap memory pool will result. + When `dispose()` is called, if the object has a property named `ondispose` + then it is treated as follows: + - If it is a function, it is called with the struct object as its `this`. + That method must not throw - if it does, the exception will be + ignored. + - If it is an array, it may contain functions, pointers, and/or JS + strings. If an entry is a function, it is called as described + above. If it's a number, it's assumed to be a pointer and is + passed to the `dealloc()` function configured for the parent + [StructBinder][]. If it's a JS string, it's assumed to be a + helpful description of the next entry in the list and is simply + ignored. Strings are supported primarily for use as debugging + information. + - Some struct APIs will manipulate the `ondispose` member, creating + it as an array or converting it from a function to array as + needed. + +- `lookupMember(memberName,throwIfNotFound=true)` + Given the name of a mapped struct member, it returns the member + description object. If not found, it either throws (if the 2nd + argument is true) or returns `undefined` (if the second argument is + false). The first argument may be either the member name as it is + mapped in the struct description or that same name with the + configured `memberPrefix` and `memberSuffix` applied, noting that + the lookup in the former case is faster.\ + This method may be called directly on the prototype, without a + struct instance. + +- `memberToJsString(memberName)` + Uses `this.lookupMember(memberName,true)` to look up the given + member. If its signature is `s` then it is assumed to refer to a + NUL-terminated, UTF-8-encoded string and its memory is decoded as + such. If its signature is not one of those then an exception is + thrown. If its address is 0, `null` is returned. See also: + `setMemberCString()`. + +- `memberIsString(memberName [,throwIfNotFound=true])` + Uses `this.lookupMember(memberName,throwIfNotFound)` to look up the + given member. Returns the member description object if the member + has a signature of `s`, else returns false. If the given member is + not found, it throws if the 2nd argument is true, else it returns + false. + +- `memberKey(string)` + Works identically to `StructBinder.StructType.memberKey()`. + +- `memberKeys()` + Returns an array of the names of the properties of this object + which refer to C-side struct counterparts. + +- `memberSignature(memberName [,emscriptenFormat=false])` + Returns the signature for a given a member property, either in this + framework's format or, if passed a truthy 2nd argument, in a format + suitable for the 2nd argument to Emscripten's `addFunction()`. + Throws if the first argument does not resolve to a struct-bound + member name. The member name is resolved using `this.lookupMember()` + and throws if the member is found mapped. + +- `memoryDump()` + Returns a Uint8Array which contains the current state of this + object's raw memory buffer. Potentially useful for debugging, but + not much else. Note that the memory is necessarily, for + compatibility with C, written in the host platform's endianness and + is thus not useful as a persistent/portable serialization format. + +- `setMemberCString(memberName,str)` + Uses `StructType.allocCString()` to allocate a new C-style string, + assign it to the given member, and add the new string to this + object's `ondispose` list for cleanup when `this.dispose()` is + called. This function throws if `lookupMember()` fails for the given + member name, if allocation of the string fails, or if the member has + a signature value of anything other than `s`. Returns `this`. + *Achtung*: calling this repeatedly will not immediately free the + previous values because this code cannot know whether they are in + use in other places, namely C. Instead, each time this is called, + the prior value is retained in the `ondispose` list for cleanup when + the struct is disposed of. Because of the complexities and general + uncertainties of memory ownership and lifetime in such + constellations, it is recommended that the use of C-string members + from JS be kept to a minimum or that the relationship be one-way: + let C manage the strings and only fetch them from JS using, e.g., + `memberToJsString()`. + + + +API: Struct Constructors +------------------------------------------------------------ + +Struct constructors (the functions returned from [StructBinder][]) +are used for, intuitively enough, creating new instances of a given +struct type: + +> +``` +const x = new MyStruct; +``` + +Normally they should be passed no arguments, but they optionally +accept a single argument: a WASM heap pointer address of memory +which the object will use for storage. It does _not_ take over +ownership of that memory and that memory must be valid at +for least as long as this struct instance. This is used, for example, +to proxy static/shared C-side instances: + +> +``` +const x = new MyStruct( someCFuncWhichReturnsAMyStructPointer() ); +... +x.dispose(); // does NOT free the memory +``` + +The JS-side construct does not own the memory in that case and has no +way of knowing when the C-side struct is destroyed. Results are +specifically undefined if the JS-side struct is used after the C-side +struct's member is freed. + +> Potential TODO: add a way of passing ownership of the C-side struct +to the JS-side object. e.g. maybe simply pass `true` as the second +argument to tell the constructor to take over ownership. Currently the +pointer can be taken over using something like +`myStruct.ondispose=[myStruct.pointer]` immediately after creation. + +These constructors have the following "static" members: + +- `disposeAll()` + For each instance of this struct, the equivalent of its `dispose()` + method is called. This frees all WASM-allocated memory associated + with _all_ instances and clears the `instanceForPointer()` + mappings. Returns `this`. + +- `instanceForPointer(pointer)` + Given a pointer value (accessible via the `pointer` property of all + struct instances) which ostensibly refers to an instance of this + class, this returns the instance associated with it, or `undefined` + if no object _of this specific struct type_ is mapped to that + pointer. When C-side code calls back into JS code and passes a + pointer to an object, this function can be used to type-safely + "cast" that pointer back to its original object. + +- `isA(value)` + Returns true if its argument was created by this constructor. + +- `memberKey(string)` + Works exactly as documented for [StructType][]. + +- `memberKeys(string)` + Works exactly as documented for [StructType][]. + +- `resolveToInstance(value [,throwIfNot=false])` + Works like `instanceForPointer()` but accepts either an instance + of this struct type or a pointer which resolves to one. + It returns an instance of this struct type on success. + By default it returns a falsy value if its argument is not, + or does not resolve to, an instance of this struct type, + but if passed a truthy second argument then it will throw + instead. + +- `structInfo` + The structure description passed to [StructBinder][] when this + constructor was generated. + +- `structName` + The structure name passed to [StructBinder][] when this constructor + was generated. + + + +API: Struct Prototypes +------------------------------------------------------------ + +The prototypes of structs created via [the constructors described in +the previous section][StructCtors] are each a struct-type-specific +instance of [StructType][] and add the following struct-type-specific +properties to the mix: + +- `structInfo` + The struct description metadata, as it was given to the + [StructBinder][] which created this class. + +- `structName` + The name of the struct, as it was given to the [StructBinder][] which + created this class. + + +API: Struct Instances +------------------------------------------------------------------------ + +Instances of structs created via [the constructors described +above][StructCtors] each have the following instance-specific state in +common: + +- `pointer` + A read-only numeric property which is the "pointer" returned by the + configured allocator when this object is constructed. After + `dispose()` (inherited from [StructType][]) is called, this property + has the `undefined` value. When passing instances of this struct to + C-bound code, `pointer` is the value which must be passed in place + of a C-side struct pointer. When calling C-side code which takes a + pointer to a struct of this type, simply pass it `myStruct.pointer`. + + +Appendices +============================================================ + + +Appendix A: Limitations, TODOs, and Non-TODOs +------------------------------------------------------------ + +- This library only supports the basic set of member types supported + by WASM: numbers (which includes pointers). Nested structs are not + handled except that a member may be a _pointer_ to such a + struct. Whether or not it ever will depends entirely on whether its + developer ever needs that support. Conversion of strings between + JS and C requires infrastructure specific to each WASM environment + and is not directly supported by this library. + +- Binding functions to struct instances, such that C can see and call + JS-defined functions, is not as transparent as it really could be, + due to [shortcomings in the Emscripten + `addFunction()`/`removeFunction()` + interfaces](https://github.com/emscripten-core/emscripten/issues/17323). Until + a replacement for that API can be written, this support will be + quite limited. It _is_ possible to bind a JS-defined function to a + C-side function pointer and call that function from C. What's + missing is easier-to-use/more transparent support for doing so. + - In the meantime, a [standalone + subproject](/file/common/whwasmutil.js) of Jaccwabyt provides such a + binding mechanism, but integrating it directly with Jaccwabyt would + not only more than double its size but somehow feels inappropriate, so + experimentation is in order for how to offer that capability via + completely optional [StructBinderFactory][] config options. + +- It "might be interesting" to move access of the C-bound members into + a sub-object. e.g., from JS they might be accessed via + `myStructInstance.s.structMember`. The main advantage is that it would + eliminate any potential confusion about which members are part of + the C struct and which exist purely in JS. "The problem" with that + is that it requires internally mapping the `s` member back to the + object which contains it, which makes the whole thing more costly + and adds one more moving part which can break. Even so, it's + something to try out one rainy day. Maybe even make it optional and + make the `s` name configurable via the [StructBinderFactory][] + options. (Over-engineering is an arguably bad habit of mine.) + +- It "might be interesting" to offer (de)serialization support. It + would be very limited, e.g. we can't serialize arbitrary pointers in + any meaningful way, but "might" be useful for structs which contain + only numeric or C-string state. As it is, it's easy enough for + client code to write wrappers for that and handle the members in + ways appropriate to their apps. Any impl provided in this library + would have the shortcoming that it may inadvertently serialize + pointers (since they're just integers), resulting in potential chaos + after deserialization. Perhaps the struct description can be + extended to tag specific members as serializable and how to + serialize them. + + +Appendix D: Debug Info +------------------------------------------------------------ + +The [StructBinderFactory][], [StructBinder][], and [StructType][] classes +all have the following "unsupported" method intended primarily +to assist in their own development, as opposed to being for use in +client code: + +- `debugFlags(flags)` (integer) + An "unsupported" debugging option which may change or be removed at + any time. Its argument is a set of flags to enable/disable certain + debug/tracing output for property accessors: 0x01 for getters, 0x02 + for setters, 0x04 for allocations, 0x08 for deallocations. Pass 0 to + disable all flags and pass a negative value to _completely_ clear + all flags. The latter has the side effect of telling the flags to be + inherited from the next-higher-up class in the hierarchy, with + [StructBinderFactory][] being top-most, followed by [StructBinder][], then + [StructType][]. + + + +Appendix G: Generating Struct Descriptions From C +------------------------------------------------------------ + +Struct definitions are _ideally_ generated from WASM-compiled C, as +opposed to simply guessing the sizeofs and offsets, so that the sizeof +and offset information can be collected using C's `sizeof()` and +`offsetof()` features (noting that struct padding may impact offsets +in ways which might not be immediately obvious, so writing them by +hand is _most certainly not recommended_). + +How exactly the desciption is generated is necessarily +project-dependent. It's tempting say, "oh, that's easy! We'll just +write it by hand!" but that would be folly. The struct sizes and byte +offsets into the struct _must_ be precisely how C-side code sees the +struct or the runtime results are completely undefined. + +The approach used in developing and testing _this_ software is... + +Below is a complete copy/pastable example of how we can use a small +set of macros to generate struct descriptions from C99 or later into +static string memory. Simply add such a file to your WASM build, +arrange for its function to be exported[^export-func], and call it +from JS (noting that it requires environment-specific JS glue to +convert the returned pointer to a JS-side string). Use `JSON.parse()` +to process it, then feed the included struct descriptions into the +binder factory at your leisure. + +------------------------------------------------------------ + +```c +#include /* memset() */ +#include /* offsetof() */ +#include /* snprintf() */ +#include /* int64_t */ +#include + +struct ExampleStruct { + int v4; + void * ppV; + int64_t v8; + void (*xFunc)(void*); +}; +typedef struct ExampleStruct ExampleStruct; + +const char * wasm__ctype_json(void){ + static char strBuf[512 * 8] = {0} + /* Static buffer which must be sized large enough for + our JSON. The string-generation macros try very + hard to assert() if this buffer is too small. */; + int n = 0, structCount = 0 /* counters for the macros */; + char * pos = &strBuf[1] + /* Write-position cursor. Skip the first byte for now to help + protect against a small race condition */; + char const * const zEnd = pos + sizeof(strBuf) + /* one-past-the-end cursor (virtual EOF) */; + if(strBuf[0]) return strBuf; // Was set up in a previous call. + + //////////////////////////////////////////////////////////////////// + // First we need to build up our macro framework... + + //////////////////////////////////////////////////////////////////// + // Core output-generating macros... +#define lenCheck assert(pos < zEnd - 100) +#define outf(format,...) \ + pos += snprintf(pos, ((size_t)(zEnd - pos)), format, __VA_ARGS__); \ + lenCheck +#define out(TXT) outf("%s",TXT) +#define CloseBrace(LEVEL) \ + assert(LEVEL<5); memset(pos, '}', LEVEL); pos+=LEVEL; lenCheck + + //////////////////////////////////////////////////////////////////// + // Macros for emiting StructBinders... +#define StructBinder__(TYPE) \ + n = 0; \ + outf("%s{", (structCount++ ? ", " : "")); \ + out("\"name\": \"" # TYPE "\","); \ + outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ + out(",\"members\": {"); +#define StructBinder_(T) StructBinder__(T) +// ^^^ extra indirection needed to expand CurrentStruct +#define StructBinder StructBinder_(CurrentStruct) +#define _StructBinder CloseBrace(2) +#define M(MEMBER,SIG) \ + outf("%s\"%s\": " \ + "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ + (n++ ? ", " : ""), #MEMBER, \ + (int)offsetof(CurrentStruct,MEMBER), \ + (int)sizeof(((CurrentStruct*)0)->MEMBER), \ + SIG) + // End of macros. + //////////////////////////////////////////////////////////////////// + + //////////////////////////////////////////////////////////////////// + // With that out of the way, we can do what we came here to do. + out("\"structs\": ["); { + +// For each struct description, do... +#define CurrentStruct ExampleStruct + StructBinder { + M(v4,"i"); + M(ppV,"p"); + M(v8,"j"); + M(xFunc,"v(p)"); + } _StructBinder; +#undef CurrentStruct + + } out( "]"/*structs*/); + //////////////////////////////////////////////////////////////////// + // Done! Finalize the output... + out("}"/*top-level wrapper*/); + *pos = 0; + strBuf[0] = '{'/*end of the race-condition workaround*/; + return strBuf; + +// If this file will ever be concatenated or #included with others, +// it's good practice to clean up our macros: +#undef StructBinder +#undef StructBinder_ +#undef StructBinder__ +#undef M +#undef _StructBinder +#undef CloseBrace +#undef out +#undef outf +#undef lenCheck +} +``` + +------------------------------------------------------------ + + + +[sqlite3]: https://sqlite.org +[emscripten]: https://emscripten.org +[sgb]: https://wanderinghorse.net/home/stephan/ +[appendix-g]: #appendix-g +[StructBinderFactory]: #api-binderfactory +[StructCtors]: #api-structctor +[StructType]: #api-structtype +[StructBinder]: #api-structbinder +[StructInstance]: #api-structinstance +[^export-func]: In Emscripten, add its name, prefixed with `_`, to the + project's `EXPORT_FUNCTIONS` list. +[BigInt64Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt64Array +[TextDecoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder +[TextEncoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder +[MDN]: https://developer.mozilla.org/docs/Web/API diff --git a/ext/wasm/jaccwabyt/jaccwabyt_test.c b/ext/wasm/jaccwabyt/jaccwabyt_test.c new file mode 100644 index 0000000000..7e2db394c6 --- /dev/null +++ b/ext/wasm/jaccwabyt/jaccwabyt_test.c @@ -0,0 +1,178 @@ +#include +#include /* memset() */ +#include /* offsetof() */ +#include /* snprintf() */ +#include /* int64_t */ +/*#include */ /* malloc/free(), needed for emscripten exports. */ +extern void * malloc(size_t); +extern void free(void *); + +/* +** 2022-06-25 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +*********************************************************************** +** +** Utility functions for use with the emscripten/WASM bits. These +** functions ARE NOT part of the sqlite3 public API. They are strictly +** for internal use by the JS/WASM bindings. +** +** This file is intended to be WASM-compiled together with sqlite3.c, +** e.g.: +** +** emcc ... sqlite3.c wasm_util.c +*/ + +/* +** Experimenting with output parameters. +*/ +int jaccwabyt_test_intptr(int * p){ + if(1==((int)p)%3){ + /* kludge to get emscripten to export malloc() and free() */; + free(malloc(0)); + } + return *p = *p * 2; +} +int64_t jaccwabyt_test_int64_max(void){ + return (int64_t)0x7fffffffffffffff; +} +int64_t jaccwabyt_test_int64_min(void){ + return ~jaccwabyt_test_int64_max(); +} +int64_t jaccwabyt_test_int64_times2(int64_t x){ + return x * 2; +} + +void jaccwabyt_test_int64_minmax(int64_t * min, int64_t *max){ + *max = jaccwabyt_test_int64_max(); + *min = jaccwabyt_test_int64_min(); + /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ +} +int64_t jaccwabyt_test_int64ptr(int64_t * p){ + /*printf("jaccwabyt_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ + return *p = *p * 2; +} + +void jaccwabyt_test_stack_overflow(int recurse){ + if(recurse) jaccwabyt_test_stack_overflow(recurse); +} + +struct WasmTestStruct { + int v4; + void * ppV; + const char * cstr; + int64_t v8; + void (*xFunc)(void*); +}; +typedef struct WasmTestStruct WasmTestStruct; +void jaccwabyt_test_struct(WasmTestStruct * s){ + if(s){ + s->v4 *= 2; + s->v8 = s->v4 * 2; + s->ppV = s; + s->cstr = __FILE__; + if(s->xFunc) s->xFunc(s); + } + return; +} + +/** For testing the 'string-free' whwasmutil.xWrap() conversion. */ +char * jaccwabyt_test_str_hello(int fail){ + char * s = fail ? 0 : (char *)malloc(6); + if(s){ + memcpy(s, "hello", 5); + s[5] = 0; + } + return s; +} + +/* +** Returns a NUL-terminated string containing a JSON-format metadata +** regarding C structs, for use with the StructBinder API. The +** returned memory is static and is only written to the first time +** this is called. +*/ +const char * jaccwabyt_test_ctype_json(void){ + static char strBuf[1024 * 8] = {0}; + int n = 0, structCount = 0, groupCount = 0; + char * pos = &strBuf[1] /* skip first byte for now to help protect + against a small race condition */; + char const * const zEnd = pos + sizeof(strBuf); + if(strBuf[0]) return strBuf; + /* Leave first strBuf[0] at 0 until the end to help guard against a + tiny race condition. If this is called twice concurrently, they + might end up both writing to strBuf, but they'll both write the + same thing, so that's okay. If we set byte 0 up front then the + 2nd instance might return a partially-populated string. */ + + //////////////////////////////////////////////////////////////////// + // First we need to build up our macro framework... + //////////////////////////////////////////////////////////////////// + // Core output macros... +#define lenCheck assert(pos < zEnd - 100) +#define outf(format,...) \ + pos += snprintf(pos, ((size_t)(zEnd - pos)), format, __VA_ARGS__); \ + lenCheck +#define out(TXT) outf("%s",TXT) +#define CloseBrace(LEVEL) \ + assert(LEVEL<5); memset(pos, '}', LEVEL); pos+=LEVEL; lenCheck + + //////////////////////////////////////////////////////////////////// + // Macros for emitting StructBinder descriptions... +#define StructBinder__(TYPE) \ + n = 0; \ + outf("%s{", (structCount++ ? ", " : "")); \ + out("\"name\": \"" # TYPE "\","); \ + outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ + out(",\"members\": {"); +#define StructBinder_(T) StructBinder__(T) +// ^^^ indirection needed to expand CurrentStruct +#define StructBinder StructBinder_(CurrentStruct) +#define _StructBinder CloseBrace(2) +#define M(MEMBER,SIG) \ + outf("%s\"%s\": " \ + "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ + (n++ ? ", " : ""), #MEMBER, \ + (int)offsetof(CurrentStruct,MEMBER), \ + (int)sizeof(((CurrentStruct*)0)->MEMBER), \ + SIG) + // End of macros + //////////////////////////////////////////////////////////////////// + + out("\"structs\": ["); { + +#define CurrentStruct WasmTestStruct + StructBinder { + M(v4,"i"); + M(cstr,"s"); + M(ppV,"p"); + M(v8,"j"); + M(xFunc,"v(p)"); + } _StructBinder; +#undef CurrentStruct + + } out( "]"/*structs*/); + out("}"/*top-level object*/); + *pos = 0; + strBuf[0] = '{'/*end of the race-condition workaround*/; + return strBuf; +#undef DefGroup +#undef Def +#undef _DefGroup +#undef StructBinder +#undef StructBinder_ +#undef StructBinder__ +#undef M +#undef _StructBinder +#undef CurrentStruct +#undef CloseBrace +#undef out +#undef outf +#undef lenCheck +} diff --git a/ext/wasm/jaccwabyt/jaccwabyt_test.exports b/ext/wasm/jaccwabyt/jaccwabyt_test.exports new file mode 100644 index 0000000000..b6182207b5 --- /dev/null +++ b/ext/wasm/jaccwabyt/jaccwabyt_test.exports @@ -0,0 +1,10 @@ +_jaccwabyt_test_intptr +_jaccwabyt_test_int64ptr +_jaccwabyt_test_int64_max +_jaccwabyt_test_int64_min +_jaccwabyt_test_int64_minmax +_jaccwabyt_test_int64_times2 +_jaccwabyt_test_struct +_jaccwabyt_test_ctype_json +_jaccwabyt_test_stack_overflow +_jaccwabyt_test_str_hello diff --git a/ext/fiddle/testing1.html b/ext/wasm/testing1.html similarity index 76% rename from ext/fiddle/testing1.html rename to ext/wasm/testing1.html index bf22f30ff3..0c64470221 100644 --- a/ext/fiddle/testing1.html +++ b/ext/wasm/testing1.html @@ -4,10 +4,9 @@ - - + + sqlite3-api.js tests -
sqlite3-api.js tests
@@ -25,9 +24,11 @@
-
Everything on this page happens in the dev console.
- - +
Most stuff on this page happens in the dev console.
+
+
+ + diff --git a/ext/wasm/testing1.js b/ext/wasm/testing1.js new file mode 100644 index 0000000000..e4b0882644 --- /dev/null +++ b/ext/wasm/testing1.js @@ -0,0 +1,1088 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic test script for sqlite3-api.js. This file must be run in + main JS thread and sqlite3.js must have been loaded before it. +*/ +'use strict'; +(function(){ + const T = self.SqliteTestUtil; + const toss = function(...args){throw new Error(args.join(' '))}; + const debug = console.debug.bind(console); + const eOutput = document.querySelector('#test-output'); + const log = console.log.bind(console) + const logHtml = function(...args){ + log.apply(this, args); + const ln = document.createElement('div'); + ln.append(document.createTextNode(args.join(' '))); + eOutput.append(ln); + }; + + const eqApprox = function(v1,v2,factor=0.05){ + //debug('eqApprox',v1, v2); + return v1>=(v2-factor) && v1<=(v2+factor); + }; + + const testBasicSanity = function(db,sqlite3){ + const capi = sqlite3.capi; + log("Basic sanity tests..."); + T.assert(Number.isInteger(db.pointer)). + mustThrowMatching(()=>db.pointer=1, /read-only/). + assert(0===capi.sqlite3_extended_result_codes(db.pointer,1)). + assert('main'===db.dbName(0)); + let pId; + let st = db.prepare( + new TextEncoder('utf-8').encode("select 3 as a") + /* Testing handling of Uint8Array input */ + ); + //debug("statement =",st); + try { + T.assert(Number.isInteger(st.pointer)) + .mustThrowMatching(()=>st.pointer=1, /read-only/) + .assert(1===db.openStatementCount()) + .assert(!st._mayGet) + .assert('a' === st.getColumnName(0)) + .assert(1===st.columnCount) + .assert(0===st.parameterCount) + .mustThrow(()=>st.bind(1,null)) + .assert(true===st.step()) + .assert(3 === st.get(0)) + .mustThrow(()=>st.get(1)) + .mustThrow(()=>st.get(0,~capi.SQLITE_INTEGER)) + .assert(3 === st.get(0,capi.SQLITE_INTEGER)) + .assert(3 === st.getInt(0)) + .assert('3' === st.get(0,capi.SQLITE_TEXT)) + .assert('3' === st.getString(0)) + .assert(3.0 === st.get(0,capi.SQLITE_FLOAT)) + .assert(3.0 === st.getFloat(0)) + .assert(3 === st.get({}).a) + .assert(3 === st.get([])[0]) + .assert(3 === st.getJSON(0)) + .assert(st.get(0,capi.SQLITE_BLOB) instanceof Uint8Array) + .assert(1===st.get(0,capi.SQLITE_BLOB).length) + .assert(st.getBlob(0) instanceof Uint8Array) + .assert('3'.charCodeAt(0) === st.getBlob(0)[0]) + .assert(st._mayGet) + .assert(false===st.step()) + .assert(!st._mayGet) + ; + pId = st.pointer; + T.assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). + assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). + assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). + assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); + }finally{ + st.finalize(); + } + T.assert(!st.pointer) + .assert(0===db.openStatementCount()); + let list = []; + db.exec({ + sql:['CREATE TABLE t(a,b);', + "INSERT INTO t(a,b) VALUES(1,2),(3,4),", + "(?,?),('blob',X'6869')"/*intentionally missing semicolon to test for + off-by-one bug in string-to-WASM conversion*/], + multi: true, + saveSql: list, + bind: [5,6] + }); + //debug("Exec'd SQL:", list); + T.assert(2 === list.length) + .assert('string'===typeof list[1]) + .assert(4===db.changes()); + if(capi.wasm.bigIntEnabled){ + T.assert(4n===db.changes(false,true)); + } + let blob = db.selectValue("select b from t where a='blob'"); + T.assert(blob instanceof Uint8Array). + assert(0x68===blob[0] && 0x69===blob[1]); + blob = null; + + let counter = 0, colNames = []; + list.length = 0; + db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ + rowMode: 'object', + resultRows: list, + columnNames: colNames, + callback: function(row,stmt){ + ++counter; + T.assert((row.a%2 && row.a<6) || 'blob'===row.a); + } + }); + T.assert(2 === colNames.length) + .assert('a' === colNames[0]) + .assert(4 === counter) + .assert(4 === list.length); + list.length = 0; + db.exec("SELECT a a, b b FROM t",{ + rowMode: 'array', + callback: function(row,stmt){ + ++counter; + T.assert(Array.isArray(row)) + .assert((0===row[1]%2 && row[1]<7) + || (row[1] instanceof Uint8Array)); + } + }); + T.assert(8 === counter); + T.assert(Number.MIN_SAFE_INTEGER === + db.selectValue("SELECT "+Number.MIN_SAFE_INTEGER)). + assert(Number.MAX_SAFE_INTEGER === + db.selectValue("SELECT "+Number.MAX_SAFE_INTEGER)); + if(capi.wasm.bigIntEnabled){ + const mI = capi.wasm.xCall('jaccwabyt_test_int64_max'); + const b = BigInt(Number.MAX_SAFE_INTEGER * 2); + T.assert(b === db.selectValue("SELECT "+b)). + assert(b === db.selectValue("SELECT ?", b)). + assert(mI == db.selectValue("SELECT $x", {$x:mI})); + }else{ + /* Curiously, the JS spec seems to be off by one with the definitions + of MIN/MAX_SAFE_INTEGER: + + https://github.com/emscripten-core/emscripten/issues/17391 */ + T.mustThrow(()=>db.selectValue("SELECT "+(Number.MAX_SAFE_INTEGER+1))). + mustThrow(()=>db.selectValue("SELECT "+(Number.MIN_SAFE_INTEGER-1))); + } + + st = db.prepare("update t set b=:b where a='blob'"); + try { + const ndx = st.getParamIndex(':b'); + T.assert(1===ndx); + st.bindAsBlob(ndx, "ima blob").reset(true); + } finally { + st.finalize(); + } + + try { + throw new capi.WasmAllocError; + }catch(e){ + T.assert(e instanceof Error) + .assert(e instanceof capi.WasmAllocError); + } + + try { + db.prepare("/*empty SQL*/"); + toss("Must not be reached."); + }catch(e){ + T.assert(e instanceof sqlite3.SQLite3Error) + .assert(0==e.message.indexOf('Cannot prepare empty')); + } + + T.assert(capi.sqlite3_errstr(capi.SQLITE_IOERR_ACCESS).indexOf("I/O")>=0). + assert(capi.sqlite3_errstr(capi.SQLITE_CORRUPT).indexOf('malformed')>0). + assert(capi.sqlite3_errstr(capi.SQLITE_OK) === 'not an error'); + + // Custom db error message handling via sqlite3_prepare_v2/v3() + if(capi.wasm.exports.sqlite3_wasm_db_error){ + log("Testing custom error message via prepare_v3()..."); + let rc = capi.sqlite3_prepare_v3(db.pointer, [/*invalid*/], -1, 0, null, null); + T.assert(capi.SQLITE_MISUSE === rc) + .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")); + log("errmsg =",capi.sqlite3_errmsg(db.pointer)); + } + }/*testBasicSanity()*/; + + const testUDF = function(db){ + db.createFunction("foo",function(a,b){return a+b}); + T.assert(7===db.selectValue("select foo(3,4)")). + assert(5===db.selectValue("select foo(3,?)",2)). + assert(5===db.selectValue("select foo(?,?2)",[1,4])). + assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5})); + db.createFunction("bar", { + arity: -1, + callback: function(){ + var rc = 0; + for(let i = 0; i < arguments.length; ++i) rc += arguments[i]; + return rc; + } + }).createFunction({ + name: "asis", + callback: (arg)=>arg + }); + + //log("Testing DB::selectValue() w/ UDF..."); + T.assert(0===db.selectValue("select bar()")). + assert(1===db.selectValue("select bar(1)")). + assert(3===db.selectValue("select bar(1,2)")). + assert(-1===db.selectValue("select bar(1,2,-4)")). + assert('hi'===db.selectValue("select asis('hi')")); + + T.assert('hi' === db.selectValue("select ?",'hi')). + assert(null===db.selectValue("select null")). + assert(null === db.selectValue("select ?",null)). + assert(null === db.selectValue("select ?",[null])). + assert(null === db.selectValue("select $a",{$a:null})). + assert(eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). + assert(eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))) + ; + + //log("Testing binding and UDF propagation of blobs..."); + let blobArg = new Uint8Array(2); + blobArg.set([0x68, 0x69], 0); + let blobRc = db.selectValue("select asis(?1)", blobArg); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length). + assert(0x68==blobRc[0] && 0x69==blobRc[1]); + blobRc = db.selectValue("select asis(X'6869')"); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length). + assert(0x68==blobRc[0] && 0x69==blobRc[1]); + + blobArg = new Int8Array(2); + blobArg.set([0x68, 0x69]); + //debug("blobArg=",blobArg); + blobRc = db.selectValue("select asis(?1)", blobArg); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length); + //debug("blobRc=",blobRc); + T.assert(0x68==blobRc[0] && 0x69==blobRc[1]); + }; + + const testAttach = function(db){ + const resultRows = []; + db.exec({ + sql:new TextEncoder('utf-8').encode([ + // ^^^ testing string-vs-typedarray handling in execMulti() + "attach 'foo.db' as foo;", + "create table foo.bar(a);", + "insert into foo.bar(a) values(1),(2),(3);", + "select a from foo.bar order by a;" + ].join('')), + multi: true, + rowMode: 0, + resultRows + }); + T.assert(3===resultRows.length) + .assert(2===resultRows[1]); + T.assert(2===db.selectValue('select a from foo.bar where a>1 order by a')); + db.exec("detach foo"); + T.mustThrow(()=>db.exec("select * from foo.bar")); + }; + + const testIntPtr = function(db,S,Module){ + const w = S.capi.wasm; + const stack = w.scopedAllocPush(); + let ptrInt; + const origValue = 512; + const ptrValType = 'i32'; + try{ + ptrInt = w.scopedAlloc(4); + w.setMemValue(ptrInt,origValue, ptrValType); + const cf = w.xGet('jaccwabyt_test_intptr'); + const oldPtrInt = ptrInt; + //log('ptrInt',ptrInt); + //log('getMemValue(ptrInt)',w.getMemValue(ptrInt)); + T.assert(origValue === w.getMemValue(ptrInt, ptrValType)); + const rc = cf(ptrInt); + //log('cf(ptrInt)',rc); + //log('ptrInt',ptrInt); + //log('getMemValue(ptrInt)',w.getMemValue(ptrInt,ptrValType)); + T.assert(2*origValue === rc). + assert(rc === w.getMemValue(ptrInt,ptrValType)). + assert(oldPtrInt === ptrInt); + const pi64 = w.scopedAlloc(8)/*ptr to 64-bit integer*/; + const o64 = 0x010203040506/*>32-bit integer*/; + const ptrType64 = 'i64'; + if(w.bigIntEnabled){ + log("BigInt support is enabled..."); + w.setMemValue(pi64, o64, ptrType64); + //log("pi64 =",pi64, "o64 = 0x",o64.toString(16), o64); + const v64 = ()=>w.getMemValue(pi64,ptrType64) + //log("getMemValue(pi64)",v64()); + T.assert(v64() == o64); + //T.assert(o64 === w.getMemValue(pi64, ptrType64)); + const cf64w = w.xGet('jaccwabyt_test_int64ptr'); + cf64w(pi64); + //log("getMemValue(pi64)",v64()); + T.assert(v64() == BigInt(2 * o64)); + cf64w(pi64); + T.assert(v64() == BigInt(4 * o64)); + + const biTimes2 = w.xGet('jaccwabyt_test_int64_times2'); + T.assert(BigInt(2 * o64) === + biTimes2(BigInt(o64)/*explicit conv. required to avoid TypeError + in the call :/ */)); + + const pMin = w.scopedAlloc(16); + const pMax = pMin + 8; + const g64 = (p)=>w.getMemValue(p,ptrType64); + w.setMemValue(pMin, 0, ptrType64); + w.setMemValue(pMax, 0, ptrType64); + const minMaxI64 = [ + w.xCall('jaccwabyt_test_int64_min'), + w.xCall('jaccwabyt_test_int64_max') + ]; + T.assert(minMaxI64[0] < BigInt(Number.MIN_SAFE_INTEGER)). + assert(minMaxI64[1] > BigInt(Number.MAX_SAFE_INTEGER)); + //log("int64_min/max() =",minMaxI64, typeof minMaxI64[0]); + w.xCall('jaccwabyt_test_int64_minmax', pMin, pMax); + T.assert(g64(pMin) === minMaxI64[0], "int64 mismatch"). + assert(g64(pMax) === minMaxI64[1], "int64 mismatch") + /* ^^^ that will fail, as of this writing, due to + mismatched getMemValue()/setMemValue() impls in the + Emscripten-generated glue. We install a + replacement getMemValue() in sqlite3-api.js to work + around that bug: + + https://github.com/emscripten-core/emscripten/issues/17322 + */; + //log("pMin",g64(pMin), "pMax",g64(pMax)); + w.setMemValue(pMin, minMaxI64[0], ptrType64); + T.assert(g64(pMin) === minMaxI64[0]). + assert(minMaxI64[0] === db.selectValue("select ?",g64(pMin))). + assert(minMaxI64[1] === db.selectValue("select ?",g64(pMax))); + const rxRange = /out of range for int64/; + T.mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[0] - BigInt(1))}, + rxRange). + mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[1] + BigInt(1))}, + (e)=>rxRange.test(e.message)); + }else{ + log("No BigInt support. Skipping related tests."); + log("\"The problem\" here is that we can manipulate, at the byte level,", + "heap memory to set 64-bit values, but we can't get those values", + "back into JS because of the lack of 64-bit number support."); + } + }finally{ + const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); + //log("x=",x,"y=",y,"z=",z); // just looking at the alignment + w.scopedAllocPop(stack); + } + }/*testIntPtr()*/; + + const testStructStuff = function(db,S,M){ + const W = S.capi.wasm, C = S; + /** Maintenance reminder: the rest of this function is copy/pasted + from the upstream jaccwabyt tests. */ + log("Jaccwabyt tests..."); + const MyStructDef = { + sizeof: 16, + members: { + p4: {offset: 0, sizeof: 4, signature: "i"}, + pP: {offset: 4, sizeof: 4, signature: "P"}, + ro: {offset: 8, sizeof: 4, signature: "i", readOnly: true}, + cstr: {offset: 12, sizeof: 4, signature: "s"} + } + }; + if(W.bigIntEnabled){ + const m = MyStructDef; + m.members.p8 = {offset: m.sizeof, sizeof: 8, signature: "j"}; + m.sizeof += m.members.p8.sizeof; + } + const StructType = C.StructBinder.StructType; + const K = C.StructBinder('my_struct',MyStructDef); + T.mustThrowMatching(()=>K(), /via 'new'/). + mustThrowMatching(()=>new K('hi'), /^Invalid pointer/); + const k1 = new K(), k2 = new K(); + try { + T.assert(k1.constructor === K). + assert(K.isA(k1)). + assert(k1 instanceof K). + assert(K.prototype.lookupMember('p4').key === '$p4'). + assert(K.prototype.lookupMember('$p4').name === 'p4'). + mustThrowMatching(()=>K.prototype.lookupMember('nope'), /not a mapped/). + assert(undefined === K.prototype.lookupMember('nope',false)). + assert(k1 instanceof StructType). + assert(StructType.isA(k1)). + assert(K.resolveToInstance(k1.pointer)===k1). + mustThrowMatching(()=>K.resolveToInstance(null,true), /is-not-a my_struct/). + assert(k1 === StructType.instanceForPointer(k1.pointer)). + mustThrowMatching(()=>k1.$ro = 1, /read-only/); + Object.keys(MyStructDef.members).forEach(function(key){ + key = K.memberKey(key); + T.assert(0 == k1[key], + "Expecting allocation to zero the memory "+ + "for "+key+" but got: "+k1[key]+ + " from "+k1.memoryDump()); + }); + T.assert('number' === typeof k1.pointer). + mustThrowMatching(()=>k1.pointer = 1, /pointer/). + assert(K.instanceForPointer(k1.pointer) === k1); + k1.$p4 = 1; k1.$pP = 2; + T.assert(1 === k1.$p4).assert(2 === k1.$pP); + if(MyStructDef.members.$p8){ + k1.$p8 = 1/*must not throw despite not being a BigInt*/; + k1.$p8 = BigInt(Number.MAX_SAFE_INTEGER * 2); + T.assert(BigInt(2 * Number.MAX_SAFE_INTEGER) === k1.$p8); + } + T.assert(!k1.ondispose); + k1.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(k1.ondispose)). + assert(k1.ondispose[0] === k1.$cstr). + assert('number' === typeof k1.$cstr). + assert('A C-string.' === k1.memberToJsString('cstr')); + k1.$pP = k2; + T.assert(k1.$pP === k2); + k1.$pP = null/*null is special-cased to 0.*/; + T.assert(0===k1.$pP); + let ptr = k1.pointer; + k1.dispose(); + T.assert(undefined === k1.pointer). + assert(undefined === K.instanceForPointer(ptr)). + mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); + const k3 = new K(); + ptr = k3.pointer; + T.assert(k3 === K.instanceForPointer(ptr)); + K.disposeAll(); + T.assert(ptr). + assert(undefined === k2.pointer). + assert(undefined === k3.pointer). + assert(undefined === K.instanceForPointer(ptr)); + }finally{ + k1.dispose(); + k2.dispose(); + } + + if(!W.bigIntEnabled){ + log("Skipping WasmTestStruct tests: BigInt not enabled."); + return; + } + + const ctype = W.xCallWrapped('jaccwabyt_test_ctype_json', 'json'); + log("Struct descriptions:",ctype.structs); + const WTStructDesc = + ctype.structs.filter((e)=>'WasmTestStruct'===e.name)[0]; + const autoResolvePtr = true /* EXPERIMENTAL */; + if(autoResolvePtr){ + WTStructDesc.members.ppV.signature = 'P'; + } + const WTStruct = C.StructBinder(WTStructDesc); + log(WTStruct.structName, WTStruct.structInfo); + const wts = new WTStruct(); + log("WTStruct.prototype keys:",Object.keys(WTStruct.prototype)); + try{ + T.assert(wts.constructor === WTStruct). + assert(WTStruct.memberKeys().indexOf('$ppV')>=0). + assert(wts.memberKeys().indexOf('$v8')>=0). + assert(!K.isA(wts)). + assert(WTStruct.isA(wts)). + assert(wts instanceof WTStruct). + assert(wts instanceof StructType). + assert(StructType.isA(wts)). + assert(wts === StructType.instanceForPointer(wts.pointer)); + T.assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). + assert(0===wts.$ppV).assert(0===wts.$xFunc). + assert(WTStruct.instanceForPointer(wts.pointer) === wts); + const testFunc = + W.xGet('jaccwabyt_test_struct'/*name gets mangled in -O3 builds!*/); + let counter = 0; + log("wts.pointer =",wts.pointer); + const wtsFunc = function(arg){ + log("This from a JS function called from C, "+ + "which itself was called from JS. arg =",arg); + ++counter; + T.assert(WTStruct.instanceForPointer(arg) === wts); + if(3===counter){ + toss("Testing exception propagation."); + } + } + wts.$v4 = 10; wts.$v8 = 20; + wts.$xFunc = W.installFunction(wtsFunc, wts.memberSignature('xFunc')) + /* ^^^ compiles wtsFunc to WASM and returns its new function pointer */; + T.assert(0===counter).assert(10 === wts.$v4).assert(20n === wts.$v8) + .assert(0 === wts.$ppV).assert('number' === typeof wts.$xFunc) + .assert(0 === wts.$cstr) + .assert(wts.memberIsString('$cstr')) + .assert(!wts.memberIsString('$v4')) + .assert(null === wts.memberToJsString('$cstr')) + .assert(W.functionEntry(wts.$xFunc) instanceof Function); + /* It might seem silly to assert that the values match + what we just set, but recall that all of those property + reads and writes are, via property interceptors, + actually marshaling their data to/from a raw memory + buffer, so merely reading them back is actually part of + testing the struct-wrapping API. */ + + testFunc(wts.pointer); + log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); + T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) + .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)) + .assert('string' === typeof wts.memberToJsString('cstr')) + .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) + .mustThrowMatching(()=>wts.memberToJsString('xFunc'), + /Invalid member type signature for C-string/) + ; + testFunc(wts.pointer); + T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) + .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)); + /** The 3rd call to wtsFunc throw from JS, which is called + from C, which is called from JS. Let's ensure that + that exception propagates back here... */ + T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); + W.uninstallFunction(wts.$xFunc); + wts.$xFunc = 0; + if(autoResolvePtr){ + wts.$ppV = 0; + T.assert(!wts.$ppV); + WTStruct.debugFlags(0x03); + wts.$ppV = wts; + T.assert(wts === wts.$ppV) + WTStruct.debugFlags(0); + } + wts.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(wts.ondispose)). + assert(wts.ondispose[0] === wts.$cstr). + assert('A C-string.' === wts.memberToJsString('cstr')); + const ptr = wts.pointer; + wts.dispose(); + T.assert(ptr).assert(undefined === wts.pointer). + assert(undefined === WTStruct.instanceForPointer(ptr)) + }finally{ + wts.dispose(); + } + }/*testStructStuff()*/; + + const testSqliteStructs = function(db,sqlite3,M){ + log("Tinkering with sqlite3_io_methods..."); + // https://www.sqlite.org/c3ref/vfs.html + // https://www.sqlite.org/c3ref/io_methods.html + const capi = sqlite3.capi, W = capi.wasm; + const sqlite3_io_methods = capi.sqlite3_io_methods, + sqlite3_vfs = capi.sqlite3_vfs, + sqlite3_file = capi.sqlite3_file; + log("struct sqlite3_file", sqlite3_file.memberKeys()); + log("struct sqlite3_vfs", sqlite3_vfs.memberKeys()); + log("struct sqlite3_io_methods", sqlite3_io_methods.memberKeys()); + + const installMethod = function callee(tgt, name, func){ + if(1===arguments.length){ + return (n,f)=>callee(tgt,n,f); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + callee.ondisposeRemoveFunc = function(){ + if(this.__ondispose){ + const who = this; + this.__ondispose.forEach( + (v)=>{ + if('number'===typeof v){ + try{capi.wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + }else{/*wasm function wrapper property*/ + delete who[v]; + } + } + ); + delete this.__ondispose; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name), + memKey = tgt.memberKey(name); + //log("installMethod",tgt, name, sigN); + if(!tgt.__ondispose){ + T.assert(undefined === tgt.ondispose); + tgt.ondispose = [callee.ondisposeRemoveFunc]; + tgt.__ondispose = []; + } + const fProxy = callee.argcProxy(func, sigN); + const pFunc = capi.wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + /** + ACHTUNG: function pointer IDs are from a different pool than + allocation IDs, starting at 1 and incrementing in steps of 1, + so if we set tgt[memKey] to those values, we'd very likely + later misinterpret them as plain old pointer addresses unless + unless we use some silly heuristic like "all values <5k are + presumably function pointers," or actually perform a function + lookup on every pointer to first see if it's a function. That + would likely work just fine, but would be kludgy. + + It turns out that "all values less than X are functions" is + essentially how it works in wasm: a function pointer is + reported to the client as its index into the + __indirect_function_table. + + So... once jaccwabyt can be told how to access the + function table, it could consider all pointer values less + than that table's size to be functions. As "real" pointer + values start much, much higher than the function table size, + that would likely work reasonably well. e.g. the object + pointer address for sqlite3's default VFS is (in this local + setup) 65104, whereas the function table has fewer than 600 + entries. + */ + const wrapperKey = '$'+memKey; + tgt[wrapperKey] = fProxy; + tgt.__ondispose.push(pFunc, wrapperKey); + //log("tgt.__ondispose =",tgt.__ondispose); + return (n,f)=>callee(tgt, n, f); + }/*installMethod*/; + + const installIOMethods = function instm(iom){ + (iom instanceof capi.sqlite3_io_methods) || toss("Invalid argument type."); + if(!instm._requireFileArg){ + instm._requireFileArg = function(arg,methodName){ + arg = capi.sqlite3_file.resolveToInstance(arg); + if(!arg){ + err("sqlite3_io_methods::xClose() was passed a non-sqlite3_file."); + } + return arg; + }; + instm._methods = { + // https://sqlite.org/c3ref/io_methods.html + xClose: /*i(P)*/function(f){ + /* int (*xClose)(sqlite3_file*) */ + log("xClose(",f,")"); + if(!(f = instm._requireFileArg(f,'xClose'))) return capi.SQLITE_MISUSE; + f.dispose(/*noting that f has externally-owned memory*/); + return 0; + }, + xRead: /*i(Ppij)*/function(f,dest,n,offset){ + /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ + log("xRead(",arguments,")"); + if(!(f = instm._requireFileArg(f))) return capi.SQLITE_MISUSE; + capi.wasm.heap8().fill(0, dest + offset, n); + return 0; + }, + xWrite: /*i(Ppij)*/function(f,dest,n,offset){ + /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ + log("xWrite(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xWrite'))) return capi.SQLITE_MISUSE; + return 0; + }, + xTruncate: /*i(Pj)*/function(f){ + /* int (*xTruncate)(sqlite3_file*, sqlite3_int64 size) */ + log("xTruncate(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xTruncate'))) return capi.SQLITE_MISUSE; + return 0; + }, + xSync: /*i(Pi)*/function(f){ + /* int (*xSync)(sqlite3_file*, int flags) */ + log("xSync(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xSync'))) return capi.SQLITE_MISUSE; + return 0; + }, + xFileSize: /*i(Pp)*/function(f,pSz){ + /* int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize) */ + log("xFileSize(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xFileSize'))) return capi.SQLITE_MISUSE; + capi.wasm.setMemValue(pSz, 0/*file size*/); + return 0; + }, + xLock: /*i(Pi)*/function(f){ + /* int (*xLock)(sqlite3_file*, int) */ + log("xLock(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xLock'))) return capi.SQLITE_MISUSE; + return 0; + }, + xUnlock: /*i(Pi)*/function(f){ + /* int (*xUnlock)(sqlite3_file*, int) */ + log("xUnlock(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xUnlock'))) return capi.SQLITE_MISUSE; + return 0; + }, + xCheckReservedLock: /*i(Pp)*/function(){ + /* int (*xCheckReservedLock)(sqlite3_file*, int *pResOut) */ + log("xCheckReservedLock(",arguments,")"); + return 0; + }, + xFileControl: /*i(Pip)*/function(){ + /* int (*xFileControl)(sqlite3_file*, int op, void *pArg) */ + log("xFileControl(",arguments,")"); + return capi.SQLITE_NOTFOUND; + }, + xSectorSize: /*i(P)*/function(){ + /* int (*xSectorSize)(sqlite3_file*) */ + log("xSectorSize(",arguments,")"); + return 0/*???*/; + }, + xDeviceCharacteristics:/*i(P)*/function(){ + /* int (*xDeviceCharacteristics)(sqlite3_file*) */ + log("xDeviceCharacteristics(",arguments,")"); + return 0; + } + }; + }/*static init*/ + iom.$iVersion = 1; + Object.keys(instm._methods).forEach( + (k)=>installMethod(iom, k, instm._methods[k]) + ); + }/*installIOMethods()*/; + + const iom = new sqlite3_io_methods, sfile = new sqlite3_file; + const err = console.error.bind(console); + try { + const IOM = sqlite3_io_methods, S3F = sqlite3_file; + //log("iom proto",iom,iom.constructor.prototype); + //log("sfile",sfile,sfile.constructor.prototype); + T.assert(0===sfile.$pMethods).assert(iom.pointer > 0); + //log("iom",iom); + /** Some of the following tests require that pMethods has a + signature of "P", as opposed to "p". */ + sfile.$pMethods = iom; + T.assert(iom === sfile.$pMethods); + sfile.$pMethods = iom.pointer; + T.assert(iom === sfile.$pMethods) + .assert(IOM.resolveToInstance(iom)) + .assert(undefined ===IOM.resolveToInstance(sfile)) + .mustThrow(()=>IOM.resolveToInstance(0,true)) + .assert(S3F.resolveToInstance(sfile.pointer)) + .assert(undefined===S3F.resolveToInstance(iom)); + T.assert(0===iom.$iVersion); + installIOMethods(iom); + T.assert(1===iom.$iVersion); + //log("iom.__ondispose",iom.__ondispose); + T.assert(Array.isArray(iom.__ondispose)).assert(iom.__ondispose.length>10); + }finally{ + iom.dispose(); + T.assert(undefined === iom.__ondispose); + } + + const dVfs = new sqlite3_vfs(capi.sqlite3_vfs_find(null)); + try { + const SB = sqlite3.StructBinder; + T.assert(dVfs instanceof SB.StructType) + .assert(dVfs.pointer) + .assert('sqlite3_vfs' === dVfs.structName) + .assert(!!dVfs.structInfo) + .assert(SB.StructType.hasExternalPointer(dVfs)) + .assert(3===dVfs.$iVersion) + .assert('number'===typeof dVfs.$zName) + .assert('number'===typeof dVfs.$xSleep) + .assert(capi.wasm.functionEntry(dVfs.$xOpen)) + .assert(dVfs.memberIsString('zName')) + .assert(dVfs.memberIsString('$zName')) + .assert(!dVfs.memberIsString('pAppData')) + .mustThrowMatching(()=>dVfs.memberToJsString('xSleep'), + /Invalid member type signature for C-string/) + .mustThrowMatching(()=>dVfs.memberSignature('nope'), /nope is not a mapped/) + .assert('string' === typeof dVfs.memberToJsString('zName')) + .assert(dVfs.memberToJsString('zName')===dVfs.memberToJsString('$zName')) + ; + log("Default VFS: @",dVfs.pointer); + Object.keys(sqlite3_vfs.structInfo.members).forEach(function(mname){ + const mk = sqlite3_vfs.memberKey(mname), mbr = sqlite3_vfs.structInfo.members[mname], + addr = dVfs[mk], prefix = 'defaultVfs.'+mname; + if(1===mbr.signature.length){ + let sep = '?', val = undefined; + switch(mbr.signature[0]){ + // TODO: move this into an accessor, e.g. getPreferredValue(member) + case 'i': case 'j': case 'f': case 'd': sep = '='; val = dVfs[mk]; break + case 'p': case 'P': sep = '@'; val = dVfs[mk]; break; + case 's': sep = '='; + //val = capi.wasm.UTF8ToString(addr); + val = dVfs.memberToJsString(mname); + break; + } + log(prefix, sep, val); + } + else{ + log(prefix," = funcptr @",addr, capi.wasm.functionEntry(addr)); + } + }); + }finally{ + dVfs.dispose(); + T.assert(undefined===dVfs.pointer); + } + }/*testSqliteStructs()*/; + + const testWasmUtil = function(DB,S){ + const w = S.capi.wasm; + /** + Maintenance reminder: the rest of this function is part of the + upstream Jaccwabyt tree. + */ + const chr = (x)=>x.charCodeAt(0); + log("heap getters..."); + { + const li = [8, 16, 32]; + if(w.bigIntEnabled) li.push(64); + for(const n of li){ + const bpe = n/8; + const s = w.heapForSize(n,false); + T.assert(bpe===s.BYTES_PER_ELEMENT). + assert(w.heapForSize(s.constructor) === s); + const u = w.heapForSize(n,true); + T.assert(bpe===u.BYTES_PER_ELEMENT). + assert(s!==u). + assert(w.heapForSize(u.constructor) === u); + } + } + + log("jstrlen()..."); + { + T.assert(3 === w.jstrlen("abc")).assert(4 === w.jstrlen("äbc")); + } + + log("jstrcpy()..."); + { + const fillChar = 10; + let ua = new Uint8Array(8), rc, + refill = ()=>ua.fill(fillChar); + refill(); + rc = w.jstrcpy("hello", ua); + T.assert(6===rc).assert(0===ua[5]).assert(chr('o')===ua[4]); + refill(); + ua[5] = chr('!'); + rc = w.jstrcpy("HELLO", ua, 0, -1, false); + T.assert(5===rc).assert(chr('!')===ua[5]).assert(chr('O')===ua[4]); + refill(); + rc = w.jstrcpy("the end", ua, 4); + //log("rc,ua",rc,ua); + T.assert(4===rc).assert(0===ua[7]). + assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); + refill(); + rc = w.jstrcpy("the end", ua, 4, -1, false); + T.assert(4===rc).assert(chr(' ')===ua[7]). + assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); + refill(); + rc = w.jstrcpy("", ua, 0, 1, true); + //log("rc,ua",rc,ua); + T.assert(1===rc).assert(0===ua[0]); + refill(); + rc = w.jstrcpy("x", ua, 0, 1, true); + //log("rc,ua",rc,ua); + T.assert(1===rc).assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 1, true); + T.assert(1===rc, 'Must not write partial multi-byte char.') + .assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 2, true); + T.assert(1===rc, 'Must not write partial multi-byte char.') + .assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 2, false); + T.assert(2===rc).assert(fillChar!==ua[1]).assert(fillChar===ua[2]); + }/*jstrcpy()*/ + + log("cstrncpy()..."); + { + w.scopedAllocPush(); + try { + let cStr = w.scopedAllocCString("hello"); + const n = w.cstrlen(cStr); + let cpy = w.scopedAlloc(n+10); + let rc = w.cstrncpy(cpy, cStr, n+10); + T.assert(n+1 === rc). + assert("hello" === w.cstringToJs(cpy)). + assert(chr('o') === w.getMemValue(cpy+n-1)). + assert(0 === w.getMemValue(cpy+n)); + let cStr2 = w.scopedAllocCString("HI!!!"); + rc = w.cstrncpy(cpy, cStr2, 3); + T.assert(3===rc). + assert("HI!lo" === w.cstringToJs(cpy)). + assert(chr('!') === w.getMemValue(cpy+2)). + assert(chr('l') === w.getMemValue(cpy+3)); + }finally{ + w.scopedAllocPop(); + } + } + + log("jstrToUintArray()..."); + { + let a = w.jstrToUintArray("hello", false); + T.assert(5===a.byteLength).assert(chr('o')===a[4]); + a = w.jstrToUintArray("hello", true); + T.assert(6===a.byteLength).assert(chr('o')===a[4]).assert(0===a[5]); + a = w.jstrToUintArray("äbä", false); + T.assert(5===a.byteLength).assert(chr('b')===a[2]); + a = w.jstrToUintArray("äbä", true); + T.assert(6===a.byteLength).assert(chr('b')===a[2]).assert(0===a[5]); + } + + log("allocCString()..."); + { + const cstr = w.allocCString("hällo, world"); + const n = w.cstrlen(cstr); + T.assert(13 === n) + .assert(0===w.getMemValue(cstr+n)) + .assert(chr('d')===w.getMemValue(cstr+n-1)); + } + + log("scopedAlloc() and friends..."); + { + const alloc = w.alloc, dealloc = w.dealloc; + w.alloc = w.dealloc = null; + T.assert(!w.scopedAlloc.level) + .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) + .mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); + w.alloc = alloc; + T.mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); + w.dealloc = dealloc; + T.mustThrowMatching(()=>w.scopedAllocPop(), /^Invalid state/) + .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) + .mustThrowMatching(()=>w.scopedAlloc.level=0, /read-only/); + const asc = w.scopedAllocPush(); + let asc2; + try { + const p1 = w.scopedAlloc(16), + p2 = w.scopedAlloc(16); + T.assert(1===w.scopedAlloc.level) + .assert(Number.isFinite(p1)) + .assert(Number.isFinite(p2)) + .assert(asc[0] === p1) + .assert(asc[1]===p2); + asc2 = w.scopedAllocPush(); + const p3 = w.scopedAlloc(16); + T.assert(2===w.scopedAlloc.level) + .assert(Number.isFinite(p3)) + .assert(2===asc.length) + .assert(p3===asc2[0]); + + const [z1, z2, z3] = w.scopedAllocPtr(3); + T.assert('number'===typeof z1).assert(z2>z1).assert(z3>z2) + .assert(0===w.getMemValue(z1,'i32'), 'allocPtr() must zero the targets') + .assert(0===w.getMemValue(z3,'i32')); + }finally{ + // Pop them in "incorrect" order to make sure they behave: + w.scopedAllocPop(asc); + T.assert(0===asc.length); + T.mustThrowMatching(()=>w.scopedAllocPop(asc), + /^Invalid state object/); + if(asc2){ + T.assert(2===asc2.length,'Should be p3 and z1'); + w.scopedAllocPop(asc2); + T.assert(0===asc2.length); + T.mustThrowMatching(()=>w.scopedAllocPop(asc2), + /^Invalid state object/); + } + } + T.assert(0===w.scopedAlloc.level); + w.scopedAllocCall(function(){ + T.assert(1===w.scopedAlloc.level); + const [cstr, n] = w.scopedAllocCString("hello, world", true); + T.assert(12 === n) + .assert(0===w.getMemValue(cstr+n)) + .assert(chr('d')===w.getMemValue(cstr+n-1)); + }); + }/*scopedAlloc()*/ + + log("xCall()..."); + { + const pJson = w.xCall('jaccwabyt_test_ctype_json'); + T.assert(Number.isFinite(pJson)).assert(w.cstrlen(pJson)>300); + } + + log("xWrap()..."); + { + //int jaccwabyt_test_intptr(int * p); + //int64_t jaccwabyt_test_int64_max(void) + //int64_t jaccwabyt_test_int64_min(void) + //int64_t jaccwabyt_test_int64_times2(int64_t x) + //void jaccwabyt_test_int64_minmax(int64_t * min, int64_t *max) + //int64_t jaccwabyt_test_int64ptr(int64_t * p) + //const char * jaccwabyt_test_ctype_json(void) + T.mustThrowMatching(()=>w.xWrap('jaccwabyt_test_ctype_json',null,'i32'), + /requires 0 arg/). + assert(w.xWrap.resultAdapter('i32') instanceof Function). + assert(w.xWrap.argAdapter('i32') instanceof Function); + let fw = w.xWrap('jaccwabyt_test_ctype_json','string'); + T.mustThrowMatching(()=>fw(1), /requires 0 arg/); + let rc = fw(); + T.assert('string'===typeof rc).assert(rc.length>300); + rc = w.xCallWrapped('jaccwabyt_test_ctype_json','*'); + T.assert(rc>0 && Number.isFinite(rc)); + rc = w.xCallWrapped('jaccwabyt_test_ctype_json','string'); + T.assert('string'===typeof rc).assert(rc.length>300); + fw = w.xWrap('jaccwabyt_test_str_hello', 'string:free',['i32']); + rc = fw(0); + T.assert('hello'===rc); + rc = fw(1); + T.assert(null===rc); + + w.xWrap.resultAdapter('thrice', (v)=>3n*BigInt(v)); + w.xWrap.argAdapter('twice', (v)=>2n*BigInt(v)); + fw = w.xWrap('jaccwabyt_test_int64_times2','thrice','twice'); + rc = fw(1); + T.assert(12n===rc); + + w.scopedAllocCall(function(){ + let pI1 = w.scopedAlloc(8), pI2 = pI1+4; + w.setMemValue(pI1, 0,'*')(pI2, 0, '*'); + let f = w.xWrap('jaccwabyt_test_int64_minmax',undefined,['i64*','i64*']); + let r1 = w.getMemValue(pI1, 'i64'), r2 = w.getMemValue(pI2, 'i64'); + T.assert(!Number.isSafeInteger(r1)).assert(!Number.isSafeInteger(r2)); + }); + } + }/*testWasmUtil()*/; + + const runTests = function(Module){ + //log("Module",Module); + const sqlite3 = Module.sqlite3, + capi = sqlite3.capi, + oo = sqlite3.oo1, + wasm = capi.wasm; + log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + log("Build options:",wasm.compileOptionUsed()); + + if(1){ + /* Let's grab those last few lines of test coverage for + sqlite3-api.js... */ + const rc = wasm.compileOptionUsed(['COMPILER']); + T.assert(1 === rc.COMPILER); + const obj = {COMPILER:undefined}; + wasm.compileOptionUsed(obj); + T.assert(1 === obj.COMPILER); + } + log("WASM heap size =",wasm.heap8().length); + //log("capi.wasm.exports.__indirect_function_table",capi.wasm.exports.__indirect_function_table); + + const wasmCtypes = wasm.ctype; + //log("wasmCtypes",wasmCtypes); + T.assert(wasmCtypes.structs[0].name==='sqlite3_vfs'). + assert(wasmCtypes.structs[0].members.szOsFile.sizeof>=4). + assert(wasmCtypes.structs[1/*sqlite3_io_methods*/ + ].members.xFileSize.offset>0); + //log(wasmCtypes.structs[0].name,"members",wasmCtypes.structs[0].members); + [ /* Spot-check a handful of constants to make sure they got installed... */ + 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', + 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', + 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE' + ].forEach(function(k){ + T.assert('number' === typeof capi[k]); + }); + [/* Spot-check a few of the WASM API methods. */ + 'alloc', 'dealloc', 'installFunction' + ].forEach(function(k){ + T.assert(capi.wasm[k] instanceof Function); + }); + + const db = new oo.DB(':memory:'), startTime = performance.now(); + try { + log("DB filename:",db.filename,db.fileName()); + const banner1 = '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', + banner2 = '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'; + [ + testWasmUtil, testBasicSanity, testUDF, + testAttach, testIntPtr, testStructStuff, + testSqliteStructs + ].forEach((f)=>{ + const t = T.counter, n = performance.now(); + logHtml(banner1,"Running",f.name+"()..."); + f(db, sqlite3, Module); + logHtml(banner2,f.name+"():",T.counter - t,'tests in',(performance.now() - n),"ms"); + }); + }finally{ + db.close(); + } + logHtml("Total Test count:",T.counter,"in",(performance.now() - startTime),"ms"); + log('capi.wasm.exports',capi.wasm.exports); + }; + + sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){ + /** Use a timeout so that we are (hopefully) out from under + the module init stack when our setup gets run. Just on + principle, not because we _need_ to be. */ + //console.debug("theModule =",theModule); + //setTimeout(()=>runTests(theModule), 0); + // ^^^ Chrome warns: "VIOLATION: setTimeout() handler took A WHOLE 50ms!" + self._MODULE = theModule /* this is only to facilitate testing from the console */ + runTests(theModule); + }); +})(); diff --git a/ext/fiddle/testing2.html b/ext/wasm/testing2.html similarity index 79% rename from ext/fiddle/testing2.html rename to ext/wasm/testing2.html index b773c4aa48..739c7f66be 100644 --- a/ext/fiddle/testing2.html +++ b/ext/wasm/testing2.html @@ -4,10 +4,9 @@ - - + + sqlite3-worker.js tests -
sqlite3-worker.js tests
@@ -25,8 +24,10 @@
-
Everything on this page happens in the dev console.
- +
Most stuff on this page happens in the dev console.
+
+
+ diff --git a/ext/wasm/testing2.js b/ext/wasm/testing2.js new file mode 100644 index 0000000000..3a279513f8 --- /dev/null +++ b/ext/wasm/testing2.js @@ -0,0 +1,340 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic test script for sqlite3-worker.js. +*/ +'use strict'; +(function(){ + const T = self.SqliteTestUtil; + const SW = new Worker("api/sqlite3-worker.js"); + const DbState = { + id: undefined + }; + const eOutput = document.querySelector('#test-output'); + const log = console.log.bind(console) + const logHtml = function(cssClass,...args){ + log.apply(this, args); + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + eOutput.append(ln); + }; + const warn = console.warn.bind(console); + const error = console.error.bind(console); + const toss = (...args)=>{throw new Error(args.join(' '))}; + /** Posts a worker message as {type:type, data:data}. */ + const wMsg = function(type,data){ + log("Posting message to worker dbId="+(DbState.id||'default')+':',data); + SW.postMessage({ + type, + dbId: DbState.id, + data, + departureTime: performance.now() + }); + return SW; + }; + + SW.onerror = function(event){ + error("onerror",event); + }; + + let startTime; + + /** + A queue for callbacks which are to be run in response to async + DB commands. See the notes in runTests() for why we need + this. The event-handling plumbing of this file requires that + any DB command which includes a `messageId` property also have + a queued callback entry, as the existence of that property in + response payloads is how it knows whether or not to shift an + entry off of the queue. + */ + const MsgHandlerQueue = { + queue: [], + id: 0, + push: function(type,callback){ + this.queue.push(callback); + return type + '-' + (++this.id); + }, + shift: function(){ + return this.queue.shift(); + } + }; + + const testCount = ()=>{ + logHtml("","Total test count:",T.counter+". Total time =",(performance.now() - startTime),"ms"); + }; + + const logEventResult = function(evd){ + logHtml(evd.errorClass ? 'error' : '', + "runOneTest",evd.messageId,"Worker time =", + (evd.workerRespondTime - evd.workerReceivedTime),"ms.", + "Round-trip event time =", + (performance.now() - evd.departureTime),"ms.", + (evd.errorClass ? evd.message : "") + ); + }; + + const runOneTest = function(eventType, eventData, callback){ + T.assert(eventData && 'object'===typeof eventData); + /* ^^^ that is for the testing and messageId-related code, not + a hard requirement of all of the Worker-exposed APIs. */ + eventData.messageId = MsgHandlerQueue.push(eventType,function(ev){ + logEventResult(ev.data); + if(callback instanceof Function){ + callback(ev); + testCount(); + } + }); + wMsg(eventType, eventData); + }; + + /** Methods which map directly to onmessage() event.type keys. + They get passed the inbound event.data. */ + const dbMsgHandler = { + open: function(ev){ + DbState.id = ev.dbId; + log("open result",ev.data); + }, + exec: function(ev){ + log("exec result",ev.data); + }, + export: function(ev){ + log("export result",ev.data); + }, + error: function(ev){ + error("ERROR from the worker:",ev.data); + logEventResult(ev.data); + }, + resultRowTest1: function f(ev){ + if(undefined === f.counter) f.counter = 0; + if(ev.data) ++f.counter; + //log("exec() result row:",ev.data); + T.assert(null===ev.data || 'number' === typeof ev.data.b); + } + }; + + /** + "The problem" now is that the test results are async. We + know, however, that the messages posted to the worker will + be processed in the order they are passed to it, so we can + create a queue of callbacks to handle them. The problem + with that approach is that it's not error-handling + friendly, in that an error can cause us to bypass a result + handler queue entry. We have to perform some extra + acrobatics to account for that. + + Problem #2 is that we cannot simply start posting events: we + first have to post an 'open' event, wait for it to respond, and + collect its db ID before continuing. If we don't wait, we may + well fire off 10+ messages before the open actually responds. + */ + const runTests2 = function(){ + const mustNotReach = ()=>{ + throw new Error("This is not supposed to be reached."); + }; + runOneTest('exec',{ + sql: ["create table t(a,b)", + "insert into t(a,b) values(1,2),(3,4),(5,6)" + ].join(';'), + multi: true, + resultRows: [], columnNames: [] + }, function(ev){ + ev = ev.data; + T.assert(0===ev.resultRows.length) + .assert(0===ev.columnNames.length); + }); + runOneTest('exec',{ + sql: 'select a a, b b from t order by a', + resultRows: [], columnNames: [], + }, function(ev){ + ev = ev.data; + T.assert(3===ev.resultRows.length) + .assert(1===ev.resultRows[0][0]) + .assert(6===ev.resultRows[2][1]) + .assert(2===ev.columnNames.length) + .assert('b'===ev.columnNames[1]); + }); + runOneTest('exec',{ + sql: 'select a a, b b from t order by a', + resultRows: [], columnNames: [], + rowMode: 'object' + }, function(ev){ + ev = ev.data; + T.assert(3===ev.resultRows.length) + .assert(1===ev.resultRows[0].a) + .assert(6===ev.resultRows[2].b) + }); + runOneTest('exec',{sql:'intentional_error'}, mustNotReach); + // Ensure that the message-handler queue survives ^^^ that error... + runOneTest('exec',{ + sql:'select 1', + resultRows: [], + //rowMode: 'array', // array is the default in the Worker interface + }, function(ev){ + ev = ev.data; + T.assert(1 === ev.resultRows.length) + .assert(1 === ev.resultRows[0][0]); + }); + runOneTest('exec',{ + sql: 'select a a, b b from t order by a', + callback: 'resultRowTest1', + rowMode: 'object' + }, function(ev){ + T.assert(3===dbMsgHandler.resultRowTest1.counter); + dbMsgHandler.resultRowTest1.counter = 0; + }); + runOneTest('exec',{ + multi: true, + sql:[ + 'pragma foreign_keys=0;', + // ^^^ arbitrary query with no result columns + 'select a, b from t order by a desc; select a from t;' + // multi-exec only honors results from the first + // statement with result columns (regardless of whether) + // it has any rows). + ], + rowMode: 1, + resultRows: [] + },function(ev){ + const rows = ev.data.resultRows; + T.assert(3===rows.length). + assert(6===rows[0]); + }); + runOneTest('exec',{sql: 'delete from t where a>3'}); + runOneTest('exec',{ + sql: 'select count(a) from t', + resultRows: [] + },function(ev){ + ev = ev.data; + T.assert(1===ev.resultRows.length) + .assert(2===ev.resultRows[0][0]); + }); + if(0){ + // export requires reimpl. for portability reasons. + runOneTest('export',{}, function(ev){ + ev = ev.data; + T.assert('string' === typeof ev.filename) + .assert(ev.buffer instanceof Uint8Array) + .assert(ev.buffer.length > 1024) + .assert('application/x-sqlite3' === ev.mimetype); + }); + } + /***** close() tests must come last. *****/ + runOneTest('close',{unlink:true},function(ev){ + ev = ev.data; + T.assert('string' === typeof ev.filename); + }); + runOneTest('close',{unlink:true},function(ev){ + ev = ev.data; + T.assert(undefined === ev.filename); + }); + }; + + const runTests = function(){ + /** + Design decision time: all remaining tests depend on the 'open' + command having succeeded. In order to support multiple DBs, the + upcoming commands ostensibly have to know the ID of the DB they + want to talk to. We have two choices: + + 1) We run 'open' and wait for its response, which contains the + db id. + + 2) We have the Worker automatically use the current "default + db" (the one which was most recently opened) if no db id is + provided in the message. When we do this, the main thread may + well fire off _all_ of the test messages before the 'open' + actually responds, but because the messages are handled on a + FIFO basis, those after the initial 'open' will pick up the + "default" db. However, if the open fails, then all pending + messages (until next next 'open', at least) except for 'close' + will fail and we have no way of cancelling them once they've + been posted to the worker. + + We currently do (2) because (A) it's certainly the most + client-friendly thing to do and (B) it seems likely that most + apps using this API will only have a single db to work with so + won't need to juggle multiple DB ids. If we revert to (1) then + the following call to runTests2() needs to be moved into the + callback function of the runOneTest() check for the 'open' + command. Note, also, that using approach (2) does not keep the + user from instead using approach (1), noting that doing so + requires explicit handling of the 'open' message to account for + it. + */ + const waitForOpen = 1, + simulateOpenError = 0 /* if true, the remaining tests will + all barf if waitForOpen is + false. */; + logHtml('', + "Sending 'open' message and",(waitForOpen ? "" : "NOT ")+ + "waiting for its response before continuing."); + startTime = performance.now(); + runOneTest('open', { + filename:'testing2.sqlite3', + simulateError: simulateOpenError + }, function(ev){ + //log("open result",ev); + T.assert('testing2.sqlite3'===ev.data.filename) + .assert(ev.data.dbId) + .assert(ev.data.messageId); + DbState.id = ev.data.dbId; + if(waitForOpen) setTimeout(runTests2, 0); + }); + if(!waitForOpen) runTests2(); + }; + + SW.onmessage = function(ev){ + if(!ev.data || 'object'!==typeof ev.data){ + warn("Unknown sqlite3-worker message type:",ev); + return; + } + ev = ev.data/*expecting a nested object*/; + //log("main window onmessage:",ev); + if(ev.data && ev.data.messageId){ + /* We're expecting a queued-up callback handler. */ + const f = MsgHandlerQueue.shift(); + if('error'===ev.type){ + dbMsgHandler.error(ev); + return; + } + T.assert(f instanceof Function); + f(ev); + return; + } + switch(ev.type){ + case 'sqlite3-api': + switch(ev.data){ + case 'worker-ready': + log("Message:",ev); + self.sqlite3TestModule.setStatus(null); + runTests(); + return; + default: + warn("Unknown sqlite3-api message type:",ev); + return; + } + default: + if(dbMsgHandler.hasOwnProperty(ev.type)){ + try{dbMsgHandler[ev.type](ev);} + catch(err){ + error("Exception while handling db result message", + ev,":",err); + } + return; + } + warn("Unknown sqlite3-api message type:",ev); + } + }; + log("Init complete, but async init bits may still be running."); +})(); diff --git a/manifest b/manifest index 913da59c3c..8e9f19a064 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C wasm/fiddle\srefactoring\spart\s1\sof\sN:\smove\sfiddle\sapp\sfrom\sext/fiddle\sto\sext/wasm/fiddle,\swhich\sonly\scontains\sfiles\sintended\sto\sbe\spushed\sto\sthe\slive\ssite.\sDisabled\sbuild\sof\sthe\snon-fiddle\swasm\sparts,\spending\sa\slater\sstep\sof\sthe\srefactoring. -D 2022-08-10T09:36:10.232 +C wasm\srefactoring\spart\s2\sof\s(apparently)\s2:\smoved\sext/fiddle/...\sinto\sext/wasm\sand\srestructured\sthe\score\sAPI-related\sparts\sof\sthe\sJS/WASM\sconsiderably. +D 2022-08-10T11:26:08.660 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in a77d419b19eb2f806109ae2d0b81abb39a3a8659b00e528da7e27bd95c7e29fd +F Makefile.in 06385361460c98e2f2fafc4ec698d726b296d9b89b214d610531a6b4341c8279 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -55,16 +55,6 @@ F ext/expert/expert1.test 3c642a4e7bbb14f21ddab595436fb465a4733f47a0fe5b2855e1d5 F ext/expert/sqlite3expert.c 6ca30d73b9ed75bd56d6e0d7f2c962d2affaa72c505458619d0ff5d9cdfac204 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 -F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api 356c356931b58eccf68367120f304db43ab6c2ef2f62f17f12f5a99737b43c38 -F ext/fiddle/SqliteTestUtil.js 2e87d424b12674476bdf8139934dcacc3ff8a7a5f5ff4392ba5e5a8d8cee9fbd -F ext/fiddle/sqlite3-api.js 5a6cc120f3eeaab65e49bcdab234e83d83c67440e04bd97191bdc004ac0cda35 -F ext/fiddle/sqlite3-worker.js 50b7a9ce14c8fae0af965e35605fe12cafb79c1e01e99216d8110d8b02fbf4b5 -F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a -F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a -F ext/fiddle/testing1.js fbeac92a5ac22668a54fd358ffc75c275d83e505e770aa484045614cb2a6cf44 -F ext/fiddle/testing2.html 9063b2430ade2fe9da4e711addd1b51a2741cf0c7ebf6926472a5e5dd63c0bc4 -F ext/fiddle/testing2.js 7b45b4e7fddbd51dbaf89b6722c02758051b34bac5a98c11b569a7e7572f88ee -F ext/fiddle/wasm_util.c 5944e38a93d3a436a47dd7c69059844697b6a5e44499656baa6da0ffe7fe23d5 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea @@ -482,14 +472,39 @@ F ext/session/test_session.c f433f68a8a8c64b0f5bc74dc725078f12483301ad4ae8375205 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb -F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 w ext/fiddle/EXPORTED_FUNCTIONS.fiddle -F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 w ext/fiddle/EXPORTED_RUNTIME_METHODS -F ext/wasm/GNUmakefile c71257754d3f69ed19308e91c2829be98532aa27ba1feaefe53d2bf17c047dc8 w ext/fiddle/Makefile -F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 w ext/fiddle/index.md -F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f w ext/fiddle/emscripten.css -F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae w ext/fiddle/fiddle-worker.js -F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 w ext/fiddle/fiddle.html -F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 w ext/fiddle/fiddle.js +F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 +F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 +F ext/wasm/GNUmakefile 5359a37fc13b68fad2259228590450339a0c59687744edd0db7bb93d3b1ae2b1 +F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 w ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api +F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 +F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c +F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a4a06cb776c003880b +F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 +F ext/wasm/api/sqlite3-api-glue.js 82c09f49c69984009ba5af2b628e67cc26c5dd203d383cd3091d40dab4e6514b +F ext/wasm/api/sqlite3-api-oo1.js e9612cb704c0563c5d71ed2a8dccd95bf6394fa4de3115d1b978dc269c49ab02 +F ext/wasm/api/sqlite3-api-opfs.js 0a4aa8993ae54c58a7925f547ff496bd09af2e3ef80f93168edec218ab96a182 +F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb w ext/fiddle/sqlite3-api.js +F ext/wasm/api/sqlite3-api-worker.js cae932a89e48730cd850ab280963a65a96cb8b4c58bacd54ba961991a3c32f51 w ext/fiddle/sqlite3-worker.js +F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 +F ext/wasm/api/sqlite3-wasm.c 2d3e6dea54ecaa58bbb2f74a051fa65bfcf6f5e5fc96fd62f1763443f8cd36e4 +F ext/wasm/api/sqlite3-worker.js 1325ca8d40129a82531902a3a077b795db2eeaee81746e5a0c811a04b415fa7f +F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 w ext/fiddle/SqliteTestUtil.js +F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f +F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0 w ext/fiddle/testing.css +F ext/wasm/common/whwasmutil.js 3d9deda1be718e2b10e2b6b474ba6ba857d905be314201ae5b3df5eef79f66aa +F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f +F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae +F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 +F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 +F ext/wasm/jaccwabyt/jaccwabyt.js 99b424b4d467d4544e82615b58e2fe07532a898540bf9de2a985f3c21e7082b2 +F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106 +F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f +F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19 +F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 w ext/fiddle/testing1.html +F ext/wasm/testing1.js aef553114aada187eef125f5361fd1e58bf5e8e97acfa65c10cb41dd60295daa w ext/fiddle/testing1.js +F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 w ext/fiddle/testing2.html +F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc w ext/fiddle/testing2.js F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1983,8 +1998,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c3a3cb0103126210692bbeb703e7b8793974042e1fc2473be6d0a0d9b07d5770 -R 7933790875a6b3f54b530ce22fe4d8f0 +P fb4eb93080288b60815be14afd7ddbbca470ce363fa3735352ea9a558fef583e +R 1fc860ea2e3b64c464acaae9c6ad06c1 U stephan -Z f8f037574e62bc4aeafe6c26802ca165 +Z 47854f455ec6bca85db3d0b3dd61eec0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1031463cb5..e6efbffcca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb4eb93080288b60815be14afd7ddbbca470ce363fa3735352ea9a558fef583e \ No newline at end of file +27f9da4eaaff39d1d58e9ffef7ddccf1e41b3726914f754b920e3e1fb572cba6 \ No newline at end of file From 2315e834632fcdc9f0be844b7ef379a8fe704ab7 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 10 Aug 2022 13:22:44 +0000 Subject: [PATCH 128/151] wasm opfs: error handling fix for an impossible-to-reach error case. Minor cosmetic tweaks in the wasm JSON enum. FossilOrigin-Name: 683a3b937e608a5ecaf7f63f054e8a63179d67c8b2348bf843e5e68f27a369f5 --- ext/wasm/api/sqlite3-api-opfs.js | 7 +++---- ext/wasm/api/sqlite3-wasm.c | 7 +++---- manifest | 32 ++++++++++++++++---------------- manifest.uuid | 2 +- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index a04029e302..5a0af2641f 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -32,7 +32,8 @@ // FileSystemFileHandle // FileSystemFileHandle.prototype.createSyncAccessHandle self.sqlite3.postInit.push(function(self, sqlite3){ - const warn = console.warn.bind(console); + const warn = console.warn.bind(console), + error = console.error.bind(console); if(!self.importScripts || !self.FileSystemFileHandle || !self.FileSystemFileHandle.prototype.createSyncAccessHandle){ warn("OPFS not found or its sync API is not available in this environment."); @@ -44,7 +45,6 @@ self.sqlite3.postInit.push(function(self, sqlite3){ //warn('self.FileSystemFileHandle =',self.FileSystemFileHandle); //warn('self.FileSystemFileHandle.prototype =',self.FileSystemFileHandle.prototype); const toss = (...args)=>{throw new Error(args.join(' '))}; - /* This is a web worker, so init the worker-based API. */ const capi = sqlite3.capi, wasm = capi.wasm; const sqlite3_vfs = capi.sqlite3_vfs @@ -54,8 +54,7 @@ self.sqlite3.postInit.push(function(self, sqlite3){ const sqlite3_io_methods = capi.sqlite3_io_methods || toss("Missing sqlite3.capi.sqlite3_io_methods object."); const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder."); - const error = console.error.bind(console), - debug = console.debug.bind(console), + const debug = console.debug.bind(console), log = console.log.bind(console); warn("UNDER CONSTRUCTION: setting up OPFS VFS..."); diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index c27dad56ff..506eaf66ea 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -213,8 +213,7 @@ const char * sqlite3_wasm_enum_json(void){ DefGroup(blobFinalizers) { /* SQLITE_STATIC/TRANSIENT need to be handled explicitly as ** integers to avoid casting-related warnings. */ - out("\"SQLITE_STATIC\":0, " - "\"SQLITE_TRANSIENT\":-1"); + out("\"SQLITE_STATIC\":0, \"SQLITE_TRANSIENT\":-1"); } _DefGroup; DefGroup(udfFlags) { @@ -312,8 +311,8 @@ const char * sqlite3_wasm_enum_json(void){ ** } ** } ** - ** Detailed documentation for those bits are in an external - ** file (StackBinder.md, as of this writing). + ** Detailed documentation for those bits are in the docs for the + ** Jaccwabyt JS-side component. */ /** Macros for emitting StructBinder description. */ diff --git a/manifest b/manifest index 8e9f19a064..049352213a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm\srefactoring\spart\s2\sof\s(apparently)\s2:\smoved\sext/fiddle/...\sinto\sext/wasm\sand\srestructured\sthe\score\sAPI-related\sparts\sof\sthe\sJS/WASM\sconsiderably. -D 2022-08-10T11:26:08.660 +C wasm\sopfs:\serror\shandling\sfix\sfor\san\simpossible-to-reach\serror\scase.\sMinor\scosmetic\stweaks\sin\sthe\swasm\sJSON\senum. +D 2022-08-10T13:22:44.985 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -476,22 +476,22 @@ F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d6 F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 F ext/wasm/GNUmakefile 5359a37fc13b68fad2259228590450339a0c59687744edd0db7bb93d3b1ae2b1 F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 w ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a4a06cb776c003880b F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 F ext/wasm/api/sqlite3-api-glue.js 82c09f49c69984009ba5af2b628e67cc26c5dd203d383cd3091d40dab4e6514b F ext/wasm/api/sqlite3-api-oo1.js e9612cb704c0563c5d71ed2a8dccd95bf6394fa4de3115d1b978dc269c49ab02 -F ext/wasm/api/sqlite3-api-opfs.js 0a4aa8993ae54c58a7925f547ff496bd09af2e3ef80f93168edec218ab96a182 -F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb w ext/fiddle/sqlite3-api.js -F ext/wasm/api/sqlite3-api-worker.js cae932a89e48730cd850ab280963a65a96cb8b4c58bacd54ba961991a3c32f51 w ext/fiddle/sqlite3-worker.js +F ext/wasm/api/sqlite3-api-opfs.js a899a10b83f15cace5a449dd12d957b429d1d4eb31cdb10e6ee4c22c569ece77 +F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb +F ext/wasm/api/sqlite3-api-worker.js cae932a89e48730cd850ab280963a65a96cb8b4c58bacd54ba961991a3c32f51 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 2d3e6dea54ecaa58bbb2f74a051fa65bfcf6f5e5fc96fd62f1763443f8cd36e4 +F ext/wasm/api/sqlite3-wasm.c 827357635c356ca178ebfa1be5915afacb682043d27c4a230c194c1ace787be4 F ext/wasm/api/sqlite3-worker.js 1325ca8d40129a82531902a3a077b795db2eeaee81746e5a0c811a04b415fa7f -F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 w ext/fiddle/SqliteTestUtil.js +F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0 w ext/fiddle/testing.css +F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0 F ext/wasm/common/whwasmutil.js 3d9deda1be718e2b10e2b6b474ba6ba857d905be314201ae5b3df5eef79f66aa F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae @@ -501,10 +501,10 @@ F ext/wasm/jaccwabyt/jaccwabyt.js 99b424b4d467d4544e82615b58e2fe07532a898540bf9d F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106 F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19 -F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 w ext/fiddle/testing1.html -F ext/wasm/testing1.js aef553114aada187eef125f5361fd1e58bf5e8e97acfa65c10cb41dd60295daa w ext/fiddle/testing1.js -F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 w ext/fiddle/testing2.html -F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc w ext/fiddle/testing2.js +F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 +F ext/wasm/testing1.js aef553114aada187eef125f5361fd1e58bf5e8e97acfa65c10cb41dd60295daa +F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 +F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1998,8 +1998,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb4eb93080288b60815be14afd7ddbbca470ce363fa3735352ea9a558fef583e -R 1fc860ea2e3b64c464acaae9c6ad06c1 +P 27f9da4eaaff39d1d58e9ffef7ddccf1e41b3726914f754b920e3e1fb572cba6 +R 4dcd52556f349f1e741495b31b7588f2 U stephan -Z 47854f455ec6bca85db3d0b3dd61eec0 +Z b4a12539a324c68c7e8d90ec6a48f714 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e6efbffcca..67a4ace64f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27f9da4eaaff39d1d58e9ffef7ddccf1e41b3726914f754b920e3e1fb572cba6 \ No newline at end of file +683a3b937e608a5ecaf7f63f054e8a63179d67c8b2348bf843e5e68f27a369f5 \ No newline at end of file From 217635f70911b98eb4ba0ccd4df646989eec2617 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 10 Aug 2022 15:29:21 +0000 Subject: [PATCH 129/151] Fix handling of columns with names that are SQL keywords in the ".expert" command. FossilOrigin-Name: 5e1b8221c385deb04a3ff5aafb2e9fc55aecc6ffc68328674e3afe56c4273e29 --- ext/expert/expert1.test | 10 ++++++++++ ext/expert/sqlite3expert.c | 4 ++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test index 73541122d8..dee4eb9ec0 100644 --- a/ext/expert/expert1.test +++ b/ext/expert/expert1.test @@ -391,6 +391,16 @@ do_setup_rec_test $tn.18.1 { SEARCH SomeObject USING COVERING INDEX SomeObject_idx_00000078 (x=?) } + +do_setup_rec_test $tn.19.0 { + CREATE TABLE t1("index"); +} { + SELECT * FROM t1 ORDER BY "index"; +} { + CREATE INDEX t1_idx_01a7214e ON t1('index'); + SCAN t1 USING COVERING INDEX t1_idx_01a7214e +} + } proc do_candidates_test {tn sql res} { diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c index f29b8a9e0b..c01feff58c 100644 --- a/ext/expert/sqlite3expert.c +++ b/ext/expert/sqlite3expert.c @@ -820,6 +820,10 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ */ static int idxIdentifierRequiresQuotes(const char *zId){ int i; + int nId = STRLEN(zId); + + if( sqlite3_keyword_check(zId, nId) ) return 1; + for(i=0; zId[i]; i++){ if( !(zId[i]=='_') && !(zId[i]>='0' && zId[i]<='9') diff --git a/manifest b/manifest index 1fec492199..dea91279e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\sname\sof\san\sindex\sto\scollide\swith\sa\stable\sin\sa\sdifferent\sschema. -D 2022-08-09T16:13:21.868 +C Fix\shandling\sof\scolumns\swith\snames\sthat\sare\sSQL\skeywords\sin\sthe\s".expert"\scommand. +D 2022-08-10T15:29:21.997 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -51,8 +51,8 @@ F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 -F ext/expert/expert1.test 3c642a4e7bbb14f21ddab595436fb465a4733f47a0fe5b2855e1d5ff900ef08e -F ext/expert/sqlite3expert.c 6ca30d73b9ed75bd56d6e0d7f2c962d2affaa72c505458619d0ff5d9cdfac204 +F ext/expert/expert1.test 95b00567ce0775126a1b788af2d055255014714ecfddc97913864d2f9266e583 +F ext/expert/sqlite3expert.c a912efbad597eafdb0ce934ebc11039f3190b2d479685d89184e107f65d856e1 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 F ext/fiddle/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 @@ -1982,8 +1982,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 60947da687d525e72e62bcddcdf4c5c819818855074cc318c9ee656e83897aef -R f4e408bf2436c98ebd7456a4b6d2719e -U drh -Z 37c870234d7cfb98492f5c9537ed0794 +P f963c2523872b59b8a7a14971f703f2eb0d021501b288597a958f6596885d0de +R f1b1ab8288a538b9d05ad6481f4e9c61 +U dan +Z 1bb598380018229fe8e4687805090be1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6ee9ccf9af..d499f9254e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f963c2523872b59b8a7a14971f703f2eb0d021501b288597a958f6596885d0de \ No newline at end of file +5e1b8221c385deb04a3ff5aafb2e9fc55aecc6ffc68328674e3afe56c4273e29 \ No newline at end of file From 778d334085106e7dd2850e7de24a1065cf87cad9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 10 Aug 2022 18:33:57 +0000 Subject: [PATCH 130/151] Remove (harmless) duplicate words in comments used to generate parts of the documentation. [forum:/forumpost/54abbe9ccd|forum post 54abbe9ccd] FossilOrigin-Name: b5e4f0db09ff7790862357df3c6fd38a2dcdfc82ff51b9f0d9470517648c844d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index dea91279e4..3cb11a7a1c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\shandling\sof\scolumns\swith\snames\sthat\sare\sSQL\skeywords\sin\sthe\s".expert"\scommand. -D 2022-08-10T15:29:21.997 +C Remove\s(harmless)\sduplicate\swords\sin\scomments\sused\sto\sgenerate\sparts\sof\sthe\ndocumentation.\s\s[forum:/forumpost/54abbe9ccd|forum\spost\s54abbe9ccd] +D 2022-08-10T18:33:57.196 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -573,7 +573,7 @@ F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 -F src/sqlite.h.in be265991edca9aea69986758b58ba81cbf5ae403fe0c4ea1d0c9df0cdc8f25ed +F src/sqlite.h.in aea48c2d62e5676f1764dc96ce7e375fea39bde0b1f9f77f9523f4404490684a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d F src/sqliteInt.h 3ae1d20f579149c18ddd995bbeffabf036cad9f4359dc2f27dc1b778d108ff35 @@ -1982,8 +1982,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f963c2523872b59b8a7a14971f703f2eb0d021501b288597a958f6596885d0de -R f1b1ab8288a538b9d05ad6481f4e9c61 -U dan -Z 1bb598380018229fe8e4687805090be1 +P 5e1b8221c385deb04a3ff5aafb2e9fc55aecc6ffc68328674e3afe56c4273e29 +R 184cb02d6eaa8739626257f31cfd353b +U drh +Z 615ed521719d927c6d7b5277ab1598a1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d499f9254e..992a568810 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e1b8221c385deb04a3ff5aafb2e9fc55aecc6ffc68328674e3afe56c4273e29 \ No newline at end of file +b5e4f0db09ff7790862357df3c6fd38a2dcdfc82ff51b9f0d9470517648c844d \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f6b49b4d88..ba9fd8baff 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6465,7 +6465,7 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** function C that is invoked prior to each autovacuum of the database ** file. ^The callback is passed a copy of the generic data pointer (P), ** the schema-name of the attached database that is being autovacuumed, -** the the size of the database file in pages, the number of free pages, +** the size of the database file in pages, the number of free pages, ** and the number of bytes per page, respectively. The callback should ** return the number of free pages that should be removed by the ** autovacuum. ^If the callback returns zero, then no autovacuum happens. @@ -6684,7 +6684,7 @@ int sqlite3_db_release_memory(sqlite3*); ** ^The soft heap limit may not be greater than the hard heap limit. ** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) ** is invoked with a value of N that is greater than the hard heap limit, -** the the soft heap limit is set to the value of the hard heap limit. +** the soft heap limit is set to the value of the hard heap limit. ** ^The soft heap limit is automatically enabled whenever the hard heap ** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and ** the soft heap limit is outside the range of 1..N, then the soft heap @@ -8979,7 +8979,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** if the application incorrectly accesses the destination [database connection] ** and so no error code is reported, but the operations may malfunction ** nevertheless. Use of the destination database connection while a -** backup is in progress might also also cause a mutex deadlock. +** backup is in progress might also cause a mutex deadlock. ** ** If running in [shared cache mode], the application must ** guarantee that the shared cache used by the destination database @@ -9407,7 +9407,7 @@ int sqlite3_wal_checkpoint_v2( */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ -#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ #define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* From f0d12d87fa252e5ca00e9a52642e203ec233dc7c Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 10 Aug 2022 18:40:43 +0000 Subject: [PATCH 131/151] Fix another harmless comment typo that causes a typo in the documentation. FossilOrigin-Name: bb084adb53386d8e52ca1e818a8322d2ec641b73fd8568cee01cc74c0ee9f265 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3cb11a7a1c..ffd7ea4a7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\s(harmless)\sduplicate\swords\sin\scomments\sused\sto\sgenerate\sparts\sof\sthe\ndocumentation.\s\s[forum:/forumpost/54abbe9ccd|forum\spost\s54abbe9ccd] -D 2022-08-10T18:33:57.196 +C Fix\sanother\sharmless\scomment\stypo\sthat\scauses\sa\stypo\sin\sthe\sdocumentation. +D 2022-08-10T18:40:43.400 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -643,7 +643,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 602fe229f32a96ceccae4f40824129669582096f7c355f53dbac156c9fecef23 F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 -F src/vdbe.c 81e2febe73f3db6d1ce5be29d30c4667d5fab9f6eae892ad7c80808f1af0c2bb +F src/vdbe.c 4e57ac969bb2252598024ee3ebcb0885cb7976f1606e2bd77975a506da93ed50 F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c 4cfbf7ec3ed60366a38655f3f10316c5a3d68f6d4d06e462f88679392611c756 @@ -1982,8 +1982,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5e1b8221c385deb04a3ff5aafb2e9fc55aecc6ffc68328674e3afe56c4273e29 -R 184cb02d6eaa8739626257f31cfd353b +P b5e4f0db09ff7790862357df3c6fd38a2dcdfc82ff51b9f0d9470517648c844d +R eaac50c574dee6cdda4672150249af55 U drh -Z 615ed521719d927c6d7b5277ab1598a1 +Z 30c21cc0e98d29fef6fd9de3bc3b538c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 992a568810..bee896b50b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5e4f0db09ff7790862357df3c6fd38a2dcdfc82ff51b9f0d9470517648c844d \ No newline at end of file +bb084adb53386d8e52ca1e818a8322d2ec641b73fd8568cee01cc74c0ee9f265 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 131a52f14d..2935d34d51 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4757,7 +4757,7 @@ seek_not_found: ** sqlite3BtreeNext() calls, then jump to This.P2, which will land just ** past the OP_IdxGT or OP_IdxGE opcode that follows the OP_SeekGE. ** -**
  • If the cursor ends up past the target row (indicating the the target +**
  • If the cursor ends up past the target row (indicating that the target ** row does not exist in the btree) then jump to SeekOP.P2. ** */ From fbfe5af903845ca42834252991d8a8f06daceac5 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 11 Aug 2022 15:45:32 +0000 Subject: [PATCH 132/151] Wasm-related doc additions and tweaks. FossilOrigin-Name: ec55a3aa44a80f661dd451b9ea567449ea4c7353901cfd44a4b50ca10f00ddbd --- Makefile.in | 5 ++ ext/wasm/api/README.md | 95 ++++++++++++++++++++++++++++++ ext/wasm/api/sqlite3-api-worker.js | 5 +- manifest | 48 +++++++-------- manifest.uuid | 2 +- 5 files changed, 127 insertions(+), 28 deletions(-) create mode 100644 ext/wasm/api/README.md diff --git a/Makefile.in b/Makefile.in index ba551f9567..8fac5a52d3 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1524,6 +1524,11 @@ sqlite3.dll: $(REAL_LIBOBJ) sqlite3.def # # fiddle/wasm section # +# Maintenance reminder: we can/should move this into the wasm-specific +# GNU Make makefile, but we currently need it here for access to +# $(SHELL_OPT). The rest of the wasm-related bits are handled via GNU +# Make in ext/wasm/... +# wasm_dir = ext/wasm wasm_dir_abs = $(TOP)/ext/wasm # ^^^ some emcc opts require absolute paths diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md new file mode 100644 index 0000000000..83986e74c1 --- /dev/null +++ b/ext/wasm/api/README.md @@ -0,0 +1,95 @@ +# sqlite3-api.js And Friends + +This is the README for the files `sqlite3-*.js`. This collection of +files is used to build a single-file distribution of the sqlite3 WASM +API. It is broken into multiple files because: + +1. To facilitate including or excluding certain components for + specific use cases. e.g. by removing `sqlite3-api-oo1.js` if the + OO#1 API is not needed. + +2. To facilitate modularizing the pieces for use in different WASM + build environments. e.g. the files `post-js-*.js` are for use with + Emscripten's `--post-js` feature, and nowhere else. + +3. Certain components must be in their own standalone files in order + to be loaded as JS Workers. + +Note that the structure described here is the current state of things, +not necessarily the "final" state. + +The overall idea is that the following files get concatenated +together, in the listed order, the resulting file is loaded by a +browser client: + +- `post-js-header.js`\ + Emscripten-specific header for the `--post-js` input. +- `sqlite3-api-prologue.js`\ + Contains the initial bootstrap setup of the sqlite3 API + objects. This is exposed as a function, rather than objects, so that + the next step can pass in a config object which abstracts away parts + of the WASM environment, to facilitate plugging it in to arbitrary + WASM toolchains. +- `whwasmutil.js`\ + A semi-third-party collection of JS/WASM utility code intended to + replace much of the Emscripten glue. The sqlite3 APIs internally use + these APIs instead of their Emscripten counterparts, in order to be + more portable to arbitrary WASM toolchains. This API is + configurable, in principle, for use with arbitrary WASM + toolchains. It is "semi-third-party" in that it was created in order + to support this tree but is standalone and maintained together + with... +- `jaccwabyt.js`\ + Another semi-third-party API which creates bindings between JS + and C structs, such that changes to the struct state from either JS + or C are visible to the other end of the connection. This is also an + independent spinoff project, conceived for the sqlite3 project but + maintained separately. +- `sqlite3-api-glue.js`\ + Invokes the function exposed by `sqlite3-api-prologue.js`, passing + it a configuration object to configure it for the current WASM + toolchain (noting that it currently requires Emscripten), then + removes that function from the global scope. The result of this file + is a global-scope `sqlite3` object which acts as a namespace for the + API's functionality. This object gets removed from the global scope + after the following files have attached their own features to it. +- `sqlite3-api-oo1.js`\ + Provides a high-level object-oriented wrapper to the lower-level C + API, colloquially known as OO API #1. Its API is similar to other + high-level sqlite3 JS wrappers and should feel relatively familiar + to anyone familiar with such APIs. That said, it is not a "required + component" and can be elided from builds which do not want it. +- `sqlite3-api-worker.js`\ + A Worker-thread-based API which uses OO API #1 to provide an + interface to a database which can be driven from the main Window + thread via the Worker message-passing interface. Like OO API #1, + this is an optional component, offering one of any number of + potential implementations for such an API. + - `sqlite3-worker.js`\ + Is not part of the amalgamated sources and is intended to be + loaded by a client Worker thread. It loads the sqlite3 module + and runs the Worker API which is implemented in + `sqlite3-api-worker.js`. +- `sqlite3-api-opfs.js`\ + is an in-development/experimental sqlite3 VFS wrapper, the goal of + which being to use Google Chrome's Origin-Private FileSystem (OPFS) + storage layer to provide persistent storage for database files in a + browser. It is far from complete. +- `sqlite3-api-cleanup.js`\ + the previous files temporarily create global objects in order to + communicate their state to the files which follow them, and _this_ + file connects any final components together and cleans up those + globals. As of this writing, this code ensures that the previous + files leave no global symbols installed, and it moves the sqlite3 + namespace object into the in-scope Emscripten module. Abstracting + this for other WASM toolchains is TODO. +- `post-js-footer.js`\ + Emscripten-specific footer for the `--post-js` input. This closes + off the lexical scope opened by `post-js-header.js`. + +The build process glues those files together, resulting in +`sqlite3-api.js`, which is everything except for the `post-js-*.js` +files, and `sqlite3.js`, which is the Emscripten-generated amalgamated +output and includes the `post-js-*.js` parts, as well as the +Emscripten-provided module loading pieces. + diff --git a/ext/wasm/api/sqlite3-api-worker.js b/ext/wasm/api/sqlite3-api-worker.js index 1d13d4ed6b..3b5c1362a1 100644 --- a/ext/wasm/api/sqlite3-api-worker.js +++ b/ext/wasm/api/sqlite3-api-worker.js @@ -84,10 +84,9 @@ self.sqlite3.initWorkerAPI = function(){ if('function' !== typeof importScripts){ toss("Cannot initalize the sqlite3 worker API in the main thread."); } - /* This is a web worker, so init the worker-based API. */ const self = this.self; - const sqlite3 = this.sqlite3 || toss("Missing self.sqlite3 object."); - const SQLite3 = sqlite3.oo1 || toss("Missing self.sqlite3.oo1 OO API."); + const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object."); + const SQLite3 = sqlite3.oo1 || toss("Missing this.sqlite3.oo1 OO API."); const DB = SQLite3.DB; /** diff --git a/manifest b/manifest index e249b5e978..e55c66fe7e 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Merge\sin\swasm-cleanups\sbranch,\sreorganizing\sand\supdating\sthe\swasm-related\scomponents. -D 2022-08-11T09:18:09.465 +C Wasm-related\sdoc\sadditions\sand\stweaks. +D 2022-08-11T15:45:32.290 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 31f4141e5adc6f002296fc2c500803f460ca3e8b2cae53effeb021e74d7b29d7 +F Makefile.in eceb228bf7b48f961b59a508f42ffa1211bf0c4c5bc818807768cc7b187ab0c8 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -472,39 +472,40 @@ F ext/session/test_session.c f433f68a8a8c64b0f5bc74dc725078f12483301ad4ae8375205 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb -F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 w ext/fiddle/EXPORTED_FUNCTIONS.fiddle -F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 w ext/fiddle/EXPORTED_RUNTIME_METHODS -F ext/wasm/GNUmakefile 5359a37fc13b68fad2259228590450339a0c59687744edd0db7bb93d3b1ae2b1 w ext/fiddle/Makefile -F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 w ext/fiddle/index.md -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 w ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api +F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 +F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 +F ext/wasm/GNUmakefile 5359a37fc13b68fad2259228590450339a0c59687744edd0db7bb93d3b1ae2b1 +F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 +F ext/wasm/api/README.md 418939809469bf671fd715eabfe7e1a03fe83111f9dea93521436077da358274 F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a4a06cb776c003880b F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 F ext/wasm/api/sqlite3-api-glue.js 82c09f49c69984009ba5af2b628e67cc26c5dd203d383cd3091d40dab4e6514b F ext/wasm/api/sqlite3-api-oo1.js e9612cb704c0563c5d71ed2a8dccd95bf6394fa4de3115d1b978dc269c49ab02 F ext/wasm/api/sqlite3-api-opfs.js a899a10b83f15cace5a449dd12d957b429d1d4eb31cdb10e6ee4c22c569ece77 -F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb w ext/fiddle/sqlite3-api.js -F ext/wasm/api/sqlite3-api-worker.js cae932a89e48730cd850ab280963a65a96cb8b4c58bacd54ba961991a3c32f51 w ext/fiddle/sqlite3-worker.js +F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb +F ext/wasm/api/sqlite3-api-worker.js 8c5dd7b3aa8076690f49400777634161ffcb3674ebd4d3466180eea0a4499656 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 827357635c356ca178ebfa1be5915afacb682043d27c4a230c194c1ace787be4 F ext/wasm/api/sqlite3-worker.js 1325ca8d40129a82531902a3a077b795db2eeaee81746e5a0c811a04b415fa7f -F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 w ext/fiddle/SqliteTestUtil.js +F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0 w ext/fiddle/testing.css +F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0 F ext/wasm/common/whwasmutil.js 3d9deda1be718e2b10e2b6b474ba6ba857d905be314201ae5b3df5eef79f66aa -F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f w ext/fiddle/emscripten.css -F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae w ext/fiddle/fiddle-worker.js -F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 w ext/fiddle/fiddle.html -F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 w ext/fiddle/fiddle.js +F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f +F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae +F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 +F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 F ext/wasm/jaccwabyt/jaccwabyt.js 99b424b4d467d4544e82615b58e2fe07532a898540bf9de2a985f3c21e7082b2 F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106 F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19 -F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 w ext/fiddle/testing1.html -F ext/wasm/testing1.js aef553114aada187eef125f5361fd1e58bf5e8e97acfa65c10cb41dd60295daa w ext/fiddle/testing1.js -F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 w ext/fiddle/testing2.html -F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc w ext/fiddle/testing2.js +F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 +F ext/wasm/testing1.js aef553114aada187eef125f5361fd1e58bf5e8e97acfa65c10cb41dd60295daa +F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 +F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1998,9 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb084adb53386d8e52ca1e818a8322d2ec641b73fd8568cee01cc74c0ee9f265 683a3b937e608a5ecaf7f63f054e8a63179d67c8b2348bf843e5e68f27a369f5 -R 4ba76f10a568f2d09e4030f5a10b07d7 -T +closed 683a3b937e608a5ecaf7f63f054e8a63179d67c8b2348bf843e5e68f27a369f5 Closed\sby\sintegrate-merge. +P c072594d3de3d6893c5d4a9d68439b84d043325f105b0d065575765a6e66c196 +R 10881e70503a9a9d9b0dab671ab8c767 U stephan -Z 747d196ca566e8555d460e3b390787ac +Z a38a9d9f317cfa8025ab6574aa1c9e88 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8a6676aad3..aa71bebdbf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c072594d3de3d6893c5d4a9d68439b84d043325f105b0d065575765a6e66c196 \ No newline at end of file +ec55a3aa44a80f661dd451b9ea567449ea4c7353901cfd44a4b50ca10f00ddbd \ No newline at end of file From e6d1650c5b9571e734bb00a3dd73e3f3b7f7e3c9 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 12 Aug 2022 09:32:51 +0000 Subject: [PATCH 133/151] wasm: corrected (in as-yet unused code) a flags check. FossilOrigin-Name: 06b6113783dd81252a5dd125e1cddb52a9d4e387c74cfb3ba88475e47a31060a --- ext/wasm/api/sqlite3-api-opfs.js | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 5a0af2641f..4acab7770a 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -194,7 +194,7 @@ self.sqlite3.postInit.push(function(self, sqlite3){ f.$pMethods = oIom.pointer; __opfsHandles[pFile] = f; f.opfsHandle = null /* TODO */; - if(capi.SQLITE_OPEN_DELETEONCLOSE){ + if(flags & capi.SQLITE_OPEN_DELETEONCLOSE){ f.deleteOnClose = true; } f.filename = zName ? wasm.cstringToJs(zName) : randomFilename(); diff --git a/manifest b/manifest index e55c66fe7e..70cac96b86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Wasm-related\sdoc\sadditions\sand\stweaks. -D 2022-08-11T15:45:32.290 +C wasm:\scorrected\s(in\sas-yet\sunused\scode)\sa\sflags\scheck. +D 2022-08-12T09:32:51.158 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 F ext/wasm/api/sqlite3-api-glue.js 82c09f49c69984009ba5af2b628e67cc26c5dd203d383cd3091d40dab4e6514b F ext/wasm/api/sqlite3-api-oo1.js e9612cb704c0563c5d71ed2a8dccd95bf6394fa4de3115d1b978dc269c49ab02 -F ext/wasm/api/sqlite3-api-opfs.js a899a10b83f15cace5a449dd12d957b429d1d4eb31cdb10e6ee4c22c569ece77 +F ext/wasm/api/sqlite3-api-opfs.js c93cdd14f81a26b3a64990515ee05c7e29827fbc8fba4e4c2fef3a37a984db89 F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb F ext/wasm/api/sqlite3-api-worker.js 8c5dd7b3aa8076690f49400777634161ffcb3674ebd4d3466180eea0a4499656 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c072594d3de3d6893c5d4a9d68439b84d043325f105b0d065575765a6e66c196 -R 10881e70503a9a9d9b0dab671ab8c767 +P ec55a3aa44a80f661dd451b9ea567449ea4c7353901cfd44a4b50ca10f00ddbd +R 812baa6dfe982f54923133a5de75e62f U stephan -Z a38a9d9f317cfa8025ab6574aa1c9e88 +Z e1734d64415d2036a906d9b54d25edc4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aa71bebdbf..3f5ea198c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec55a3aa44a80f661dd451b9ea567449ea4c7353901cfd44a4b50ca10f00ddbd \ No newline at end of file +06b6113783dd81252a5dd125e1cddb52a9d4e387c74cfb3ba88475e47a31060a \ No newline at end of file From e1c34624844134521f2485a81ab52a78b554a824 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 12 Aug 2022 13:07:21 +0000 Subject: [PATCH 134/151] wasm: document the role of sqlite3-wasm.c. Other minor doc updates. FossilOrigin-Name: 4c10b9b18feca82440273f8192fef951ad051bbfd8aad4d3c840cf6375130afd --- ext/wasm/api/README.md | 12 +++++++++--- ext/wasm/api/sqlite3-api-worker.js | 2 +- ext/wasm/api/sqlite3-wasm.c | 9 +++++---- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 24 insertions(+), 17 deletions(-) diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 83986e74c1..025c9afc38 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -1,8 +1,9 @@ # sqlite3-api.js And Friends -This is the README for the files `sqlite3-*.js`. This collection of -files is used to build a single-file distribution of the sqlite3 WASM -API. It is broken into multiple files because: +This is the README for the files `sqlite3-*.js` and +`sqlite3-wasm.c`. This collection of files is used to build a +single-file distribution of the sqlite3 WASM API. It is broken into +multiple JS files because: 1. To facilitate including or excluding certain components for specific use cases. e.g. by removing `sqlite3-api-oo1.js` if the @@ -93,3 +94,8 @@ files, and `sqlite3.js`, which is the Emscripten-generated amalgamated output and includes the `post-js-*.js` parts, as well as the Emscripten-provided module loading pieces. +The non-JS outlier file is `sqlite3-wasm.c`: it is a proxy for +`sqlite3.c` which `#include`'s that file and adds a couple more +WASM-specific helper functions, at least one of which requires access +to private/static `sqlite3.c` internals. `sqlite3.wasm` is compiled +from this file rather than `sqlite3.c`. diff --git a/ext/wasm/api/sqlite3-api-worker.js b/ext/wasm/api/sqlite3-api-worker.js index 3b5c1362a1..95b27b21ec 100644 --- a/ext/wasm/api/sqlite3-api-worker.js +++ b/ext/wasm/api/sqlite3-api-worker.js @@ -351,7 +351,7 @@ self.sqlite3.initWorkerAPI = function(){ form: { type: apiCommand, - dbId: optional DB ID value (not currently used!) + dbId: optional DB ID value (else uses a default db handle) data: apiArguments } diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 506eaf66ea..6a81da3e5f 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -6,10 +6,11 @@ ** ** For purposes of certain hand-crafted C/Wasm function bindings, we ** need a way of reporting errors which is consistent with the rest of -** the C API. To that end, this internal-use-only function is a thin -** proxy around sqlite3ErrorWithMessage(). The intent is that it only -** be used from Wasm bindings such as sqlite3_prepare_v2/v3(), and -** definitely not from client code. +** the C API, as opposed to throwing JS exceptions. To that end, this +** internal-use-only function is a thin proxy around +** sqlite3ErrorWithMessage(). The intent is that it only be used from +** Wasm bindings such as sqlite3_prepare_v2/v3(), and definitely not +** from client code. ** ** Returns err_code. */ diff --git a/manifest b/manifest index 70cac96b86..a34ccd1ec5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm:\scorrected\s(in\sas-yet\sunused\scode)\sa\sflags\scheck. -D 2022-08-12T09:32:51.158 +C wasm:\sdocument\sthe\srole\sof\ssqlite3-wasm.c.\sOther\sminor\sdoc\supdates. +D 2022-08-12T13:07:21.618 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -478,7 +478,7 @@ F ext/wasm/GNUmakefile 5359a37fc13b68fad2259228590450339a0c59687744edd0db7bb93d3 F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md 418939809469bf671fd715eabfe7e1a03fe83111f9dea93521436077da358274 +F ext/wasm/api/README.md b3c502bda9a7da0621edcbbd055b9c498a0cfed09ffed59aa026461e6e514ed1 F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a4a06cb776c003880b F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 @@ -486,9 +486,9 @@ F ext/wasm/api/sqlite3-api-glue.js 82c09f49c69984009ba5af2b628e67cc26c5dd203d383 F ext/wasm/api/sqlite3-api-oo1.js e9612cb704c0563c5d71ed2a8dccd95bf6394fa4de3115d1b978dc269c49ab02 F ext/wasm/api/sqlite3-api-opfs.js c93cdd14f81a26b3a64990515ee05c7e29827fbc8fba4e4c2fef3a37a984db89 F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb -F ext/wasm/api/sqlite3-api-worker.js 8c5dd7b3aa8076690f49400777634161ffcb3674ebd4d3466180eea0a4499656 +F ext/wasm/api/sqlite3-api-worker.js 1124f404ecdf3c14d9f829425cef778cd683911a9883f0809a463c3c7773c9fd F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 827357635c356ca178ebfa1be5915afacb682043d27c4a230c194c1ace787be4 +F ext/wasm/api/sqlite3-wasm.c 8585793ca8311c7a0618b7e00ed2b3729799c20664a51f196258576e3d475c9e F ext/wasm/api/sqlite3-worker.js 1325ca8d40129a82531902a3a077b795db2eeaee81746e5a0c811a04b415fa7f F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec55a3aa44a80f661dd451b9ea567449ea4c7353901cfd44a4b50ca10f00ddbd -R 812baa6dfe982f54923133a5de75e62f +P 06b6113783dd81252a5dd125e1cddb52a9d4e387c74cfb3ba88475e47a31060a +R a62abfa1bd7be911f96cf33e0210940f U stephan -Z e1734d64415d2036a906d9b54d25edc4 +Z 65285bd6e6db388430367f6dfb4bd451 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f5ea198c4..2d231af8ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06b6113783dd81252a5dd125e1cddb52a9d4e387c74cfb3ba88475e47a31060a \ No newline at end of file +4c10b9b18feca82440273f8192fef951ad051bbfd8aad4d3c840cf6375130afd \ No newline at end of file From 5b0e63eb4c58a9ceb5c6d0d817ab34e75b02c001 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 12 Aug 2022 17:55:18 +0000 Subject: [PATCH 135/151] Minor wasm-related doc clarification and remove an obsolete code comment. FossilOrigin-Name: 1b1f650a08da93da97ed3a96b9a3e4eac567472c266188c02a9bffe1cf620e53 --- ext/wasm/api/README.md | 4 ++-- ext/wasm/testing1.js | 12 ++---------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 12 insertions(+), 20 deletions(-) diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 025c9afc38..43d2b0dd5d 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -31,7 +31,7 @@ browser client: the next step can pass in a config object which abstracts away parts of the WASM environment, to facilitate plugging it in to arbitrary WASM toolchains. -- `whwasmutil.js`\ +- `../common/whwasmutil.js`\ A semi-third-party collection of JS/WASM utility code intended to replace much of the Emscripten glue. The sqlite3 APIs internally use these APIs instead of their Emscripten counterparts, in order to be @@ -40,7 +40,7 @@ browser client: toolchains. It is "semi-third-party" in that it was created in order to support this tree but is standalone and maintained together with... -- `jaccwabyt.js`\ +- `../jaccwabyt/jaccwabyt.js`\ Another semi-third-party API which creates bindings between JS and C structs, such that changes to the struct state from either JS or C are visible to the other end of the connection. This is also an diff --git a/ext/wasm/testing1.js b/ext/wasm/testing1.js index e4b0882644..a733156e7a 100644 --- a/ext/wasm/testing1.js +++ b/ext/wasm/testing1.js @@ -325,15 +325,7 @@ //log("int64_min/max() =",minMaxI64, typeof minMaxI64[0]); w.xCall('jaccwabyt_test_int64_minmax', pMin, pMax); T.assert(g64(pMin) === minMaxI64[0], "int64 mismatch"). - assert(g64(pMax) === minMaxI64[1], "int64 mismatch") - /* ^^^ that will fail, as of this writing, due to - mismatched getMemValue()/setMemValue() impls in the - Emscripten-generated glue. We install a - replacement getMemValue() in sqlite3-api.js to work - around that bug: - - https://github.com/emscripten-core/emscripten/issues/17322 - */; + assert(g64(pMax) === minMaxI64[1], "int64 mismatch"); //log("pMin",g64(pMin), "pMax",g64(pMax)); w.setMemValue(pMin, minMaxI64[0], ptrType64); T.assert(g64(pMin) === minMaxI64[0]). @@ -348,7 +340,7 @@ log("No BigInt support. Skipping related tests."); log("\"The problem\" here is that we can manipulate, at the byte level,", "heap memory to set 64-bit values, but we can't get those values", - "back into JS because of the lack of 64-bit number support."); + "back into JS because of the lack of 64-bit integer support."); } }finally{ const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); diff --git a/manifest b/manifest index a34ccd1ec5..ef72de5c1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm:\sdocument\sthe\srole\sof\ssqlite3-wasm.c.\sOther\sminor\sdoc\supdates. -D 2022-08-12T13:07:21.618 +C Minor\swasm-related\sdoc\sclarification\sand\sremove\san\sobsolete\scode\scomment. +D 2022-08-12T17:55:18.237 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -478,7 +478,7 @@ F ext/wasm/GNUmakefile 5359a37fc13b68fad2259228590450339a0c59687744edd0db7bb93d3 F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md b3c502bda9a7da0621edcbbd055b9c498a0cfed09ffed59aa026461e6e514ed1 +F ext/wasm/api/README.md b6d0fb64bfdf7bf9ce6938ea4104228f6f5bbef600f5d910b2f8c8694195988c F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a4a06cb776c003880b F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 @@ -503,7 +503,7 @@ F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356a F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19 F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 -F ext/wasm/testing1.js aef553114aada187eef125f5361fd1e58bf5e8e97acfa65c10cb41dd60295daa +F ext/wasm/testing1.js cba7134901a965743fa9289d82447ab71de4690b1ee5d06f6cb83e8b569d7943 F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 06b6113783dd81252a5dd125e1cddb52a9d4e387c74cfb3ba88475e47a31060a -R a62abfa1bd7be911f96cf33e0210940f +P 4c10b9b18feca82440273f8192fef951ad051bbfd8aad4d3c840cf6375130afd +R 3e055b525bbc2100d8c4c001b630489b U stephan -Z 65285bd6e6db388430367f6dfb4bd451 +Z 4d766400503717808861a1e631c17801 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2d231af8ed..911760f4e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c10b9b18feca82440273f8192fef951ad051bbfd8aad4d3c840cf6375130afd \ No newline at end of file +1b1f650a08da93da97ed3a96b9a3e4eac567472c266188c02a9bffe1cf620e53 \ No newline at end of file From 087f143f87928dfd3aadc2d09464905028ee5deb Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 12 Aug 2022 18:46:01 +0000 Subject: [PATCH 136/151] Add the (undocumented) SQLITE_DEFAULT_UNIX_VFS compile-time option. FossilOrigin-Name: 49828bdec5f926cd18a069d39a5db0b1e1f3528a2affcfbaa1cf7b98aca51b3b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 5 +++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ef72de5c1d..36aa4eb89d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\swasm-related\sdoc\sclarification\sand\sremove\san\sobsolete\scode\scomment. -D 2022-08-12T17:55:18.237 +C Add\sthe\s(undocumented)\sSQLITE_DEFAULT_UNIX_VFS\scompile-time\soption. +D 2022-08-12T18:46:01.170 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -572,7 +572,7 @@ F src/os.c b1c4f2d485961e9a5b6b648c36687d25047c252222e9660b7cc25a6e1ea436ab F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 2df2b33db88f00af13805d4573ee126bc5973f9e3b91d03c575fa7ba64e7dc41 +F src/os_unix.c 81e24bcad127faf32da162a4bb06e7a4604bea76fe7b1ed5b01742e9eb320554 F src/os_win.c a8ea80037e81127ca01959daa87387cc135f325c88dc745376c4f760de852a10 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 74596fc3d5d8a50de32c37225fc300cccd5ea27ea303c5f4b845d6572f999c5f @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4c10b9b18feca82440273f8192fef951ad051bbfd8aad4d3c840cf6375130afd -R 3e055b525bbc2100d8c4c001b630489b -U stephan -Z 4d766400503717808861a1e631c17801 +P 1b1f650a08da93da97ed3a96b9a3e4eac567472c266188c02a9bffe1cf620e53 +R 9329740380d01abc186d0aa46eefc155 +U drh +Z 7c0cd408e2c2d1d7e9656e705ea4f22c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 911760f4e7..4f2bf325a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b1f650a08da93da97ed3a96b9a3e4eac567472c266188c02a9bffe1cf620e53 \ No newline at end of file +49828bdec5f926cd18a069d39a5db0b1e1f3528a2affcfbaa1cf7b98aca51b3b \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index b933de3765..f809eca9fe 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -8049,7 +8049,12 @@ int sqlite3_os_init(void){ /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ +#ifdef SQLITE_DEFAULT_UNIX_VFS + sqlite3_vfs_register(&aVfs[i], + 0==strcmp(aVfs[i].zName,SQLITE_DEFAULT_UNIX_VFS)); +#else sqlite3_vfs_register(&aVfs[i], i==0); +#endif } unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); From 9c5e1e40e3f8aea2ba61e55060d7f4cd720f832e Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 15 Aug 2022 12:26:26 +0000 Subject: [PATCH 137/151] In the documentation emphasize that the use of shared cache is discouraged. Fix test cases so that they all work with shared cache disabled. FossilOrigin-Name: 52c3ae063e133c5f88cad2c3084cf46218f865ac817ab937e3529d2c6a7af300 --- manifest | 36 ++++++++++++++++++------------------ manifest.uuid | 2 +- src/sqlite.h.in | 8 ++++++++ test/attachmalloc.test | 23 ++++++++++++----------- test/e_uri.test | 2 ++ test/enc3.test | 1 - test/fts4merge4.test | 2 +- test/shared9.test | 2 +- test/sharedA.test | 5 +++++ test/sharedB.test | 5 +++++ test/thread001.test | 1 + test/thread002.test | 2 +- test/tkt-f7b4edec.test | 1 + test/uri.test | 2 ++ test/walshared.test | 2 +- 15 files changed, 59 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index 36aa4eb89d..e0b62ac483 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s(undocumented)\sSQLITE_DEFAULT_UNIX_VFS\scompile-time\soption. -D 2022-08-12T18:46:01.170 +C In\sthe\sdocumentation\semphasize\sthat\sthe\suse\sof\sshared\scache\sis\sdiscouraged.\nFix\stest\scases\sso\sthat\sthey\sall\swork\swith\sshared\scache\sdisabled. +D 2022-08-15T12:26:26.113 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -590,7 +590,7 @@ F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 -F src/sqlite.h.in aea48c2d62e5676f1764dc96ce7e375fea39bde0b1f9f77f9523f4404490684a +F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d F src/sqliteInt.h 3ae1d20f579149c18ddd995bbeffabf036cad9f4359dc2f27dc1b778d108ff35 @@ -735,7 +735,7 @@ F test/attach.test 54f8e49e88d0de48f6428267a678465863d2b8f72320612f35bd5c02e240b F test/attach2.test 256bd240da1835fb8408dd59fb7ef71f8358c7a756c46662434d11d07ba3a0ce F test/attach3.test c59d92791070c59272e00183b7353eeb94915976 F test/attach4.test 00e754484859998d124d144de6d114d920f2ed6ca2f961e6a7f4183c714f885e -F test/attachmalloc.test 12c4f028e570acf9e0a4b0b7fe6f536e21f3d5ebddcece423603d0569beaf438 +F test/attachmalloc.test 67309af95c6b765c13e7d2279d7fccbef78e6eb0565d75d51cefd5dc88784549 F test/auth.test 0f246deec5cb2f6f893f8fbb76628f182c08fe40f178b254dd72467ca012f657 F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 76d20a7fa136d63bcfcf8bcb65c0b1455ed71078d81f22bcd0550d3eb18594ab @@ -921,7 +921,7 @@ F test/e_select.test 89fa483f68d868f1be3d6f56180ed42d979979c266e051bf8b5e66a251e F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test c927f7499dc3aa28b9b556b7d6d115a2f0fe41f012b128d16bf1f3b30e9b41e4 F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 -F test/e_uri.test a3d30c7d6d7700150055b6660ee42da7f05dee2c150ce08bf6a9ededc26468cc +F test/e_uri.test 86564382132d9c453845eeb5293c7e375487b625900ab56c181a0464908417d8 F test/e_vacuum.test 89fc48e8beee2f9dfd6de1fbb2edea6542dae9121dc0fbe6313764169e742104 F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 @@ -930,7 +930,7 @@ F test/e_walhook.test 01b494287ba9e60b70f6ebf3c6c62e0ffe01788e344a4846b08e5de0b3 F test/emptytable.test a38110becbdfa6325cd65cb588dca658cd885f62 F test/enc.test 9a7be5479da985381d740b15f432800f65e2c87029ee57a318f42cb2eb43763a F test/enc2.test 848bf05f15b011719f478dddb7b5e9aea35e39e457493cba4c4eef75d849a5ec -F test/enc3.test 6807f7a7740a00361ca8d0ccd66bc60c8dc5f2b6 +F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 F test/eqp.test 473aea9599b4b7af46614b55198cd78167e4eccd48e60812a40db47c5c41dea9 F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9 @@ -1078,7 +1078,7 @@ F test/fts4lastrowid.test 185835895948d5325c7710649824042373b2203149abe8024a9319 F test/fts4merge.test e2b2ec21e287d54ec09824ccfb41e66896eeca568fc818ba0e0eb2efd94c35d2 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 -F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b +F test/fts4merge4.test 66fce89934cd9508cbdc67de486558c34912ffb2e8ffe5c9a1bbb9b8a4408ba7 F test/fts4merge5.test 69932d85cda8a1c4dcfb742865900ed8fbda51724b8cf9a45bbe226dfd06c596 F test/fts4min.test 1c11e4bde16674a0c795953509cbc3731a7d9cbd1ddc7f35467bf39d632d749f F test/fts4noti.test d5d933705b1b1516b67a5e3f8e514ecb19c6522fb3357bb744776d48427c2292 @@ -1434,9 +1434,9 @@ F test/shared4.test c75f476804e76e26bf6fa0e7b421fb0ca7d07558 F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9 F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 933ed7d71f598bb6c7a8c192a3cd30f2562fdccf514df383798599c34ffa672f -F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 -F test/sharedA.test 49d87ec54ab640fbbc3786ee3c01de94aaa482a3a9f834ad3fe92770eb69e281 -F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e +F test/shared9.test 600a257fe9d8b0272746b230e761aa1bd8802ca4cf3ba5b2136b9204f3d51efa +F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e212700 +F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test e4b4de56f454708e0747b52915135baa2cbfec4965406d6eaf02a4a5c22a9880 @@ -1526,8 +1526,8 @@ F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl d759ac44a501fb832f2ea966429ca18acfba0f9a8d34ad5c499332b079b37023 F test/testrunner.tcl 86b57135754ab2160aeb04b4829d321fb285a5cfa7a505fe61d69aed605854cc -F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef -F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 +F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 +F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 F test/thread004.test f51dfc3936184aaf73ee85f315224baad272a87f F test/thread005.test 50d10b5684399676174bd96c94ad4250b1a2c8b6 @@ -1595,7 +1595,7 @@ F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 F test/tkt-f67b41381a.test 9120eab5e949969a29087e01bf57ac6a52b6c06c16be41091a74c2a863ffc660 F test/tkt-f777251dc7a.test d1a8fc3eefb7a9e64d19ff24d5c8c94c34a632fb -F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 +F test/tkt-f7b4edec.test a0d9cf5023af8bfc066e71128f325fd4831c6c6761cad35e451d35c8492f5cf1 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 F test/tkt-fc62af4523.test 72825d3febdedcd5593a27989fc05accdbfc2bb4 @@ -1742,7 +1742,7 @@ F test/upsert3.test 88d7d590a1948a9cb6eac1b54b0642f67a9f35a1fc0f19b200e97d5d39e3 F test/upsert4.test 25d2a1da92f149331ae0c51ca6e3eee78189577585eab92de149900d62994fa5 F test/upsert5.test fff0dcfce73c649204543088d8e5bde01172676063ec9b8f8fc7f195abc386fe F test/upsertfault.test f21ca47740841fdb4d61acfa7b17646d773e67724fe8c185b71c018db8a94b35 -F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568 +F test/uri.test c1abaaaa28e9422d61e5f3f9cbc8ef993ec49fe802f581520731708561d49384 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 9fde0bb5d3a821594aa68c6829ab9c5453a084384137ebb9f6153e2d678039da @@ -1816,7 +1816,7 @@ F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c F test/walro2.test 33955a6fd874dd9724005e17f77fef89d334b3171454a1256fe4941a96766cdc F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 F test/walsetlk.test 34c901443b31ab720afc463f5b236c86ca5c4134402573dce91aa0761de8db5a -F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 +F test/walshared.test 42e3808582504878af237ea02c42ca793e8a0efaa19df7df26ac573370dbc7a3 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1b1f650a08da93da97ed3a96b9a3e4eac567472c266188c02a9bffe1cf620e53 -R 9329740380d01abc186d0aa46eefc155 +P 49828bdec5f926cd18a069d39a5db0b1e1f3528a2affcfbaa1cf7b98aca51b3b +R 138b56807af43017f46d2fa433baa3f1 U drh -Z 7c0cd408e2c2d1d7e9656e705ea4f22c +Z 713a84bc1904cad175917d5abd266782 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4f2bf325a9..fc125e1c0d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49828bdec5f926cd18a069d39a5db0b1e1f3528a2affcfbaa1cf7b98aca51b3b \ No newline at end of file +52c3ae063e133c5f88cad2c3084cf46218f865ac817ab937e3529d2c6a7af300 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ba9fd8baff..d862c4fc00 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3424,6 +3424,9 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); **
    The database is opened [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ +** The [use of shared cache mode is discouraged] and hence shared cache +** capabilities may be omitted from many builds of SQLite. In such cases, +** this option is a no-op. ** ** ^(
    [SQLITE_OPEN_PRIVATECACHE]
    **
    The database is opened [shared cache] disabled, overriding @@ -6586,6 +6589,11 @@ void *sqlite3_update_hook( ** to the same database. Sharing is enabled if the argument is true ** and disabled if the argument is false.)^ ** +** This interface is omitted if SQLite is compiled with +** [-DSQLITE_OMIT_SHARED_CACHE]. The [-DSQLITE_OMIT_SHARED_CACHE] +** compile-time option is recommended because the +** [use of shared cache mode is discouraged]. +** ** ^Cache sharing is enabled and disabled for an entire process. ** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). ** In prior versions of SQLite, diff --git a/test/attachmalloc.test b/test/attachmalloc.test index 7a82f41f64..3c6a075b43 100644 --- a/test/attachmalloc.test +++ b/test/attachmalloc.test @@ -61,17 +61,18 @@ do_malloc_test attachmalloc-2 -tclprep { ATTACH 'test2.db' AS db1; } -set enable_shared_cache [sqlite3_enable_shared_cache 1] -sqlite3 dbaux test3.db -dbaux eval {SELECT * FROM sqlite_master} -do_malloc_test attachmalloc-3 -sqlbody { - SELECT * FROM sqlite_master; - ATTACH 'test3.db' AS three; -} -cleanup { - db eval { DETACH three } +ifcapable shared_cache { + set enable_shared_cache [sqlite3_enable_shared_cache 1] + sqlite3 dbaux test3.db + dbaux eval {SELECT * FROM sqlite_master} + do_malloc_test attachmalloc-3 -sqlbody { + SELECT * FROM sqlite_master; + ATTACH 'test3.db' AS three; + } -cleanup { + db eval { DETACH three } + } + dbaux close + sqlite3_enable_shared_cache $enable_shared_cache } -dbaux close -sqlite3_enable_shared_cache $enable_shared_cache - finish_test diff --git a/test/e_uri.test b/test/e_uri.test index 92a3df78d2..b1fb47989c 100644 --- a/test/e_uri.test +++ b/test/e_uri.test @@ -358,6 +358,7 @@ foreach {tn uri error} " do_test 10.$tn { open_uri_error $uri } $error } +ifcapable shared_cache { # EVIDENCE-OF: R-23027-03515 Setting it to "shared" is equivalent to # setting the SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed # to sqlite3_open_v2(). @@ -430,6 +431,7 @@ foreach {tn uri flags shared_default isshared} { db close } sqlite3_enable_shared_cache $orig +} ;# End ifcapable shared_chache # EVIDENCE-OF: R-63472-46769 Specifying an unknown parameter in the # query component of a URI is not an error. diff --git a/test/enc3.test b/test/enc3.test index 7ede2b716f..c9d39cbf3a 100644 --- a/test/enc3.test +++ b/test/enc3.test @@ -13,7 +13,6 @@ # The focus of this file is testing of the proper handling of conversions # to the native text representation. # -# $Id: enc3.test,v 1.8 2008/01/22 01:48:09 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/fts4merge4.test b/test/fts4merge4.test index 038e460d0e..12328c23df 100644 --- a/test/fts4merge4.test +++ b/test/fts4merge4.test @@ -16,7 +16,7 @@ source $testdir/tester.tcl source $testdir/fts3_common.tcl set ::testprefix fts4merge4 -ifcapable !fts3 { +ifcapable !fts3||!shared_cache { finish_test return } diff --git a/test/shared9.test b/test/shared9.test index 1982a593e3..5bb46c2686 100644 --- a/test/shared9.test +++ b/test/shared9.test @@ -19,7 +19,7 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl set testprefix shared9 -ifcapable !view||!trigger { +ifcapable !view||!trigger||!shared_cache { finish_test return } diff --git a/test/sharedA.test b/test/sharedA.test index 55ed5749bb..ef57c89cdf 100644 --- a/test/sharedA.test +++ b/test/sharedA.test @@ -19,6 +19,11 @@ if {[run_thread_tests]==0} { finish_test ; return } db close set ::testprefix sharedA +ifcapable !shared_cache { + finish_test + return +} + if {[atomic_batch_write test.db]} { finish_test return diff --git a/test/sharedB.test b/test/sharedB.test index 1b15755076..2a7e49cd91 100644 --- a/test/sharedB.test +++ b/test/sharedB.test @@ -24,6 +24,11 @@ if {[run_thread_tests]==0} { finish_test ; return } db close set ::testprefix sharedB +ifcapable !shared_cache { + finish_test + return +} + set ::enable_shared_cache [sqlite3_enable_shared_cache 1] #------------------------------------------------------------------------- diff --git a/test/thread001.test b/test/thread001.test index 7f21fb0738..1390f0d991 100644 --- a/test/thread001.test +++ b/test/thread001.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl if {[run_thread_tests]==0} { finish_test ; return } +ifcapable !shared_cache { finish_test ; return } set ::enable_shared_cache [sqlite3_enable_shared_cache] diff --git a/test/thread002.test b/test/thread002.test index b39c9ae7cd..3e9439e213 100644 --- a/test/thread002.test +++ b/test/thread002.test @@ -19,7 +19,7 @@ set testdir [file dirname $argv0] set do_not_use_codec 1 source $testdir/tester.tcl if {[run_thread_tests]==0} { finish_test ; return } - +ifcapable !shared_cache { finish_test ; return } db close set ::enable_shared_cache [sqlite3_enable_shared_cache 1] diff --git a/test/tkt-f7b4edec.test b/test/tkt-f7b4edec.test index f6d3d5b95a..d21da31917 100644 --- a/test/tkt-f7b4edec.test +++ b/test/tkt-f7b4edec.test @@ -16,6 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !shared_cache { finish_test ; return } # Open two database connections to the same database file in # shared cache mode. Create update hooks that will fire on diff --git a/test/uri.test b/test/uri.test index 0931a25e51..2b388c4007 100644 --- a/test/uri.test +++ b/test/uri.test @@ -194,6 +194,7 @@ foreach {tn mode create_ok write_ok readonly_ok} { } $A($readonly_ok) } +ifcapable shared_cache { set orig [sqlite3_enable_shared_cache] foreach {tn options sc_default is_shared} { 1 "" 1 1 @@ -223,6 +224,7 @@ foreach {tn options sc_default is_shared} { db2 close } +} ;# end ifcapable shared_cache do_test 4.3.1 { list [catch {sqlite3 db "file:test.db?mode=rc"} msg] $msg diff --git a/test/walshared.test b/test/walshared.test index fbbdeb4de3..ccac9484c6 100644 --- a/test/walshared.test +++ b/test/walshared.test @@ -16,7 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !wal {finish_test ; return } +ifcapable !wal||!shared_cache {finish_test ; return } db close set ::enable_shared_cache [sqlite3_enable_shared_cache 1] From 5187c955f67c429c7f01a8f3b4b892ec6b534fd0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 15 Aug 2022 18:36:08 +0000 Subject: [PATCH 138/151] The new --script option to dbtotxt.c looks for an SQL script at the head of the file and generates a CLI script that will construct the database then run the SQL. FossilOrigin-Name: 6170e638ebeb12cc40c3247324237978401f701afc270de88ad03e183e82cefc --- manifest | 12 ++++----- manifest.uuid | 2 +- tool/dbtotxt.c | 71 ++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index e0b62ac483..ff9202b070 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sdocumentation\semphasize\sthat\sthe\suse\sof\sshared\scache\sis\sdiscouraged.\nFix\stest\scases\sso\sthat\sthey\sall\swork\swith\sshared\scache\sdisabled. -D 2022-08-15T12:26:26.113 +C The\snew\s--script\soption\sto\sdbtotxt.c\slooks\sfor\san\sSQL\sscript\sat\sthe\shead\sof\nthe\sfile\sand\sgenerates\sa\sCLI\sscript\sthat\swill\sconstruct\sthe\sdatabase\sthen\srun\nthe\sSQL. +D 2022-08-15T18:36:08.339 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1904,7 +1904,7 @@ F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/dbhash.c 5da0c61032d23d74f2ab84ffc5740f0e8abec94f2c45c0b4306be7eb3ae96df0 -F tool/dbtotxt.c 2620a354d109b70d71bf758d2a64e042d74e0aaa5bd6e447900ed3542410d7f9 +F tool/dbtotxt.c ca48d34eaca6d6b6e4bd6a7be2b72caf34475869054240244c60fa7e69a518d6 F tool/dbtotxt.md c9a57af8739957ef36d2cfad5c4b1443ff3688ed33e4901ee200c8b651f43f3c F tool/enlargedb.c 3e8b2612b985cfa7e3e8800031ee191b43ae80de96abb5abbd5eada62651ee21 F tool/extract-sqlite3h.tcl 069ceab0cee26cba99952bfa08c0b23e35941c837acabe143f0c355d96c9e2eb x @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 49828bdec5f926cd18a069d39a5db0b1e1f3528a2affcfbaa1cf7b98aca51b3b -R 138b56807af43017f46d2fa433baa3f1 +P 52c3ae063e133c5f88cad2c3084cf46218f865ac817ab937e3529d2c6a7af300 +R e96f02106a09b713c72886888af2923e U drh -Z 713a84bc1904cad175917d5abd266782 +Z 30df68295258a2292c6be3adf651edab # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fc125e1c0d..3f4443f6bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52c3ae063e133c5f88cad2c3084cf46218f865ac817ab937e3529d2c6a7af300 \ No newline at end of file +6170e638ebeb12cc40c3247324237978401f701afc270de88ad03e183e82cefc \ No newline at end of file diff --git a/tool/dbtotxt.c b/tool/dbtotxt.c index a56416dcc4..fbd6e3d519 100644 --- a/tool/dbtotxt.c +++ b/tool/dbtotxt.c @@ -13,8 +13,15 @@ ** dbtotxt [OPTIONS] FILENAME ** ** where OPTIONS are zero or more of: -** --pagesize N => setting the database page size for later readin -** --for-cli => prepending '.open --hexdb' to the output +** +** --for-cli prepending '.open --hexdb' to the output +** +** --script The input file is expected to start with a +** zero-terminated SQL string. Output the +** ".open --hexdb" header, then the database +** then the SQL. +** +** --pagesize N set the database page size for later reading ** ** The translation of the database appears on standard output. If the ** --pagesize command-line option is omitted, then the page size is taken @@ -43,17 +50,20 @@ static int allZero(unsigned char *aLine){ int main(int argc, char **argv){ int pgsz = 0; /* page size */ int forCli = 0; /* whether to prepend with .open */ + int bSQL = 0; /* Expect and SQL prefix */ long szFile; /* Size of the input file in bytes */ FILE *in; /* Input file */ + int nSQL; /* Number of bytes of script */ int i, j; /* Loop counters */ int nErr = 0; /* Number of errors */ const char *zInputFile = 0; /* Name of the input file */ const char *zBaseName = 0; /* Base name of the file */ int lastPage = 0; /* Last page number shown */ int iPage; /* Current page number */ - unsigned char aLine[16]; /* A single line of the file */ - unsigned char aHdr[100]; /* File header */ - unsigned char bShow[256]; /* Characters ok to display */ + unsigned char *aData = 0; /* All data */ + unsigned char *aLine; /* A single line of the file */ + unsigned char *aHdr; /* File header */ + unsigned char bShow[256]; /* Characters ok to display */ memset(bShow, '.', sizeof(bShow)); for(i=' '; i<='~'; i++){ if( i!='{' && i!='}' && i!='"' && i!='\\' ) bShow[i] = (unsigned char)i; @@ -75,6 +85,10 @@ int main(int argc, char **argv){ }else if( strcmp(z,"for-cli")==0 ){ forCli = 1; continue; + }else if( strcmp(z,"script")==0 ){ + forCli = 1; + bSQL = 1; + continue; } fprintf(stderr, "Unknown option: %s\n", argv[i]); nErr++; @@ -90,7 +104,8 @@ int main(int argc, char **argv){ nErr++; } if( nErr ){ - fprintf(stderr, "Usage: %s [--pagesize N] FILENAME\n", argv[0]); + fprintf(stderr, + "Usage: %s [--pagesize N] [--script] [--for-cli] FILENAME\n", argv[0]); exit(1); } in = fopen(zInputFile, "rb"); @@ -105,11 +120,32 @@ int main(int argc, char **argv){ fprintf(stderr, "File too short. Minimum size is 100 bytes.\n"); exit(1); } - if( fread(aHdr, 100, 1, in)!=1 ){ - fprintf(stderr, "Cannot read file header\n"); + aData = malloc( szFile+16 ); + if( aData==0 ){ + fprintf(stderr, "Failed to allocate %ld bytes\n", szFile); exit(1); } - rewind(in); + if( fread(aData, szFile, 1, in)!=1 ){ + fprintf(stderr, "Cannot read file info memory\n"); + exit(1); + } + memset(aData+szFile, 0, 16); + fclose(in); + if( bSQL ){ + for(i=0; i0 ){ + printf("%s\n", aData); + } + free( aData ); return 0; } From 207cb6f6701d78559244ed1cf4c419d7ed4e3671 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 15 Aug 2022 19:23:15 +0000 Subject: [PATCH 139/151] Extra test for SQLITE_MAX_COMPOUND_SELECT. FossilOrigin-Name: c271096736889530f392ff37e631a77f3bc9c46b290dbda245fa05249f4410fc --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/sqllimits1.test | 13 +++++++++++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ff9202b070..bb54c41f76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\snew\s--script\soption\sto\sdbtotxt.c\slooks\sfor\san\sSQL\sscript\sat\sthe\shead\sof\nthe\sfile\sand\sgenerates\sa\sCLI\sscript\sthat\swill\sconstruct\sthe\sdatabase\sthen\srun\nthe\sSQL. -D 2022-08-15T18:36:08.339 +C Extra\stest\sfor\sSQLITE_MAX_COMPOUND_SELECT. +D 2022-08-15T19:23:15.090 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1486,7 +1486,7 @@ F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 F test/spellfix4.test 51c7c26514ade169855c66bcf130bd5acfb4d7fd090cc624645ab275ae6a41fb F test/sqldiff1.test 182058e09c7082de5c6a470ff9c291337bbeb650052c2cc68fbb3d7e25861d91 -F test/sqllimits1.test f46f405d754e702227dcdf5b73f94fffcd48c676e845afa37329e4ce2e32ccbf +F test/sqllimits1.test 69d110987dbdb4bea9dbc3f151c93b4697ae6cd7e6c3a519119dc1140c8607fd F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a F test/startup.c 1beb5ca66fcc0fce95c3444db9d1674f90fc605499a574ae2434dcfc10d22805 F test/stat.test 123212a20ceb496893d5254a5f6c76442ce549fdc08d1702d8288a2bbaac8408 @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 52c3ae063e133c5f88cad2c3084cf46218f865ac817ab937e3529d2c6a7af300 -R e96f02106a09b713c72886888af2923e -U drh -Z 30df68295258a2292c6be3adf651edab +P 6170e638ebeb12cc40c3247324237978401f701afc270de88ad03e183e82cefc +R 5fdd415975dfc9f1efaf302c20079f22 +U dan +Z 538d1647acf283a5b618d8240e7b7cf4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f4443f6bf..800ff0f5cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6170e638ebeb12cc40c3247324237978401f701afc270de88ad03e183e82cefc \ No newline at end of file +c271096736889530f392ff37e631a77f3bc9c46b290dbda245fa05249f4410fc \ No newline at end of file diff --git a/test/sqllimits1.test b/test/sqllimits1.test index 6b7d7b6095..1c3f66b7fa 100644 --- a/test/sqllimits1.test +++ b/test/sqllimits1.test @@ -910,4 +910,17 @@ do_catchsql_test sqllimits1-17.1 " CREATE TABLE $nm (x PRIMARY KEY) " {1 {string or blob too big}} +#------------------------------------------------------------------------- +# +sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 10 +do_catchsql_test sqllimits1-18.1 { + CREATE TABLE b1(x); + INSERT INTO b1 VALUES(1), (2), (3), (4), (5), (6), (7), (8), (9), (10), (11); +} {0 {}} + +do_catchsql_test sqllimits1-18.2 { + INSERT INTO b1 VALUES(1), (2), (3), (4), (5), (6), (7), (8), (9), (10) + UNION VALUES(11); +} {1 {too many terms in compound SELECT}} + finish_test From 9113c87ef3ee81a19088dad11ab556201c88560d Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 16 Aug 2022 00:04:40 +0000 Subject: [PATCH 140/151] Replace the RC4-based PRNG with one based on ChaCha20. 3x faster. FossilOrigin-Name: 084d8776fa95c75440530028171c56547a341c9a952ba2f29bb533b538603c78 --- manifest | 17 ++++++----- manifest.uuid | 2 +- src/random.c | 83 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 63 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index bb54c41f76..28553e1e9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\stest\sfor\sSQLITE_MAX_COMPOUND_SELECT. -D 2022-08-15T19:23:15.090 +C Replace\sthe\sRC4-based\sPRNG\swith\sone\sbased\son\sChaCha20.\s\s3x\sfaster. +D 2022-08-16T00:04:40.434 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -585,7 +585,7 @@ F src/pragma.c 6637d624c37a8909d3edfa9d7cf694d79b49d2a0827d8c52ef15dceb641783fa F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 -F src/random.c 097dc8b31b8fba5a9aca1697aeb9fd82078ec91be734c16bffda620ced7ab83c +F src/random.c ff4b47317d22c9b6af581e25cb9243190082c57daaf175ae93c4adaa223a1e5e F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc @@ -1999,8 +1999,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6170e638ebeb12cc40c3247324237978401f701afc270de88ad03e183e82cefc -R 5fdd415975dfc9f1efaf302c20079f22 -U dan -Z 538d1647acf283a5b618d8240e7b7cf4 +P c271096736889530f392ff37e631a77f3bc9c46b290dbda245fa05249f4410fc +R 169c772b3aaee1590ba8a07f447292d5 +T *branch * chacha20-prng +T *sym-chacha20-prng * +T -sym-trunk * +U drh +Z 1e027b4a09350d19e88468e502211941 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 800ff0f5cb..3e2548ba94 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c271096736889530f392ff37e631a77f3bc9c46b290dbda245fa05249f4410fc \ No newline at end of file +084d8776fa95c75440530028171c56547a341c9a952ba2f29bb533b538603c78 \ No newline at end of file diff --git a/src/random.c b/src/random.c index 87f9e2cecb..11c5a42b6d 100644 --- a/src/random.c +++ b/src/random.c @@ -22,16 +22,38 @@ ** This structure is the current state of the generator. */ static SQLITE_WSD struct sqlite3PrngType { - unsigned char isInit; /* True if initialized */ - unsigned char i, j; /* State variables */ - unsigned char s[256]; /* State variables */ + u32 s[16]; /* 64 bytes of chacha20 state */ + u8 out[64]; /* Output bytes */ + u8 n; /* Output bytes remaining */ } sqlite3Prng; +#define ROTL(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) +#define QR(a, b, c, d) ( \ + a += b, d ^= a, d = ROTL(d,16), \ + c += d, b ^= c, b = ROTL(b,12), \ + a += b, d ^= a, d = ROTL(d, 8), \ + c += d, b ^= c, b = ROTL(b, 7)) +static void chacha_block(u32 *out, const u32 *in){ + int i; + u32 x[16]; + memcpy(x, in, 64); + for(i=0; i<10; i++){ + QR(x[0], x[4], x[ 8], x[12]); + QR(x[1], x[5], x[ 9], x[13]); + QR(x[2], x[6], x[10], x[14]); + QR(x[3], x[7], x[11], x[15]); + QR(x[0], x[5], x[10], x[15]); + QR(x[1], x[6], x[11], x[12]); + QR(x[2], x[7], x[ 8], x[13]); + QR(x[3], x[4], x[ 9], x[14]); + } + for(i=0; i<16; i++) out[i] = x[i]+in[i]; +} + /* ** Return N random bytes. */ void sqlite3_randomness(int N, void *pBuf){ - unsigned char t; unsigned char *zBuf = pBuf; /* The "wsdPrng" macro will resolve to the pseudo-random number generator @@ -61,7 +83,7 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); if( N<=0 || pBuf==0 ){ - wsdPrng.isInit = 0; + wsdPrng.s[0] = 0; sqlite3_mutex_leave(mutex); return; } @@ -75,39 +97,38 @@ void sqlite3_randomness(int N, void *pBuf){ ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random ** number generator) not as an encryption device. */ - if( !wsdPrng.isInit ){ + if( wsdPrng.s[0]==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(0); - int i; - char k[256]; - wsdPrng.j = 0; - wsdPrng.i = 0; + static const u32 chacha20_init[] = { + 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574 + }; + memcpy(&wsdPrng.s[0], chacha20_init, 16); if( NEVER(pVfs==0) ){ - memset(k, 0, sizeof(k)); + memset(&wsdPrng.s[4], 0, 44); }else{ - sqlite3OsRandomness(pVfs, 256, k); + sqlite3OsRandomness(pVfs, 44, (char*)&wsdPrng.s[4]); } - for(i=0; i<256; i++){ - wsdPrng.s[i] = (u8)i; - } - for(i=0; i<256; i++){ - wsdPrng.j += wsdPrng.s[i] + k[i]; - t = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = wsdPrng.s[i]; - wsdPrng.s[i] = t; - } - wsdPrng.isInit = 1; + wsdPrng.s[16] = wsdPrng.s[12]; + wsdPrng.s[12] = 0; + wsdPrng.n = 0; } assert( N>0 ); - do{ - wsdPrng.i++; - t = wsdPrng.s[wsdPrng.i]; - wsdPrng.j += t; - wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = t; - t += wsdPrng.s[wsdPrng.i]; - *(zBuf++) = wsdPrng.s[t]; - }while( --N ); + while( 1 /* exit by break */ ){ + if( N<=wsdPrng.n ){ + memcpy(zBuf, &wsdPrng.out[wsdPrng.n-N], N); + wsdPrng.n -= N; + break; + } + if( wsdPrng.n>0 ){ + memcpy(zBuf, wsdPrng.out, wsdPrng.n); + N -= wsdPrng.n; + zBuf += wsdPrng.n; + } + wsdPrng.s[12]++; + chacha_block((u32*)wsdPrng.out, wsdPrng.s); + wsdPrng.n = 64; + } sqlite3_mutex_leave(mutex); } From 2756f806f2efd8a025acab1687cb5d2dc0fe284f Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 16 Aug 2022 10:52:35 +0000 Subject: [PATCH 141/151] Add assert() statements to verify that the SQLITE_OPEN_EXCLUSIVE flag is always passed to the VFS when opening a temporary file. FossilOrigin-Name: e123da49ccae61d591abded52f4721aa10f20d75935c9a3e3fe826a9b8df2317 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/memjournal.c | 2 ++ src/os.c | 1 + src/pager.c | 1 + 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bb54c41f76..abc82a7f17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\stest\sfor\sSQLITE_MAX_COMPOUND_SELECT. -D 2022-08-15T19:23:15.090 +C Add\sassert()\sstatements\sto\sverify\sthat\sthe\sSQLITE_OPEN_EXCLUSIVE\sflag\sis\salways\spassed\sto\sthe\sVFS\swhen\sopening\sa\stemporary\sfile. +D 2022-08-16T10:52:35.403 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -560,7 +560,7 @@ F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6 F src/mem5.c 5a3dbd8ac8a6501152a4fc1fcae9b0900c2d7eb0589c4ec7456fdde15725a26c F src/memdb.c c2dc88f97c410eb68a24468344b65526685e18354ddfd15906750c1eaf9dc2dd -F src/memjournal.c 8bd50ae6d9c6d34b3a96cc3b4f567f9935dc358444d872ab48901a8c11ad82a6 +F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a @@ -568,14 +568,14 @@ F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c dd2b3f1cc1863079bc1349ac0fec395a500090c4fe4e11ab775310a49f2f956d F src/mutex_w32.c caa50e1c0258ac4443f52e00fe8aaea73b6d0728bd8856bedfff822cae418541 F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6 -F src/os.c b1c4f2d485961e9a5b6b648c36687d25047c252222e9660b7cc25a6e1ea436ab +F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 81e24bcad127faf32da162a4bb06e7a4604bea76fe7b1ed5b01742e9eb320554 F src/os_win.c a8ea80037e81127ca01959daa87387cc135f325c88dc745376c4f760de852a10 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 74596fc3d5d8a50de32c37225fc300cccd5ea27ea303c5f4b845d6572f999c5f +F src/pager.c c60031c483960660853dfecf14c8e830503baab1f638ac997f0144f1bd3e1781 F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564 F src/pcache.c 084e638432c610f95aea72b8509f0845d2791293f39d1b82f0c0a7e089c3bb6b @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6170e638ebeb12cc40c3247324237978401f701afc270de88ad03e183e82cefc -R 5fdd415975dfc9f1efaf302c20079f22 +P c271096736889530f392ff37e631a77f3bc9c46b290dbda245fa05249f4410fc +R 3f7d8ed697988ef9ad8d4dda04b632a0 U dan -Z 538d1647acf283a5b618d8240e7b7cf4 +Z 207999dde59e8e595694810745e7b969 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 800ff0f5cb..75e42a2853 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c271096736889530f392ff37e631a77f3bc9c46b290dbda245fa05249f4410fc \ No newline at end of file +e123da49ccae61d591abded52f4721aa10f20d75935c9a3e3fe826a9b8df2317 \ No newline at end of file diff --git a/src/memjournal.c b/src/memjournal.c index a4c17eedd4..9343801074 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -359,6 +359,8 @@ int sqlite3JournalOpen( ){ MemJournal *p = (MemJournal*)pJfd; + assert( zName || nSpill<0 || (flags & SQLITE_OPEN_EXCLUSIVE) ); + /* Zero the file-handle object. If nSpill was passed zero, initialize ** it using the sqlite3OsOpen() function of the underlying VFS. In this ** case none of the code in this module is executed as a result of calls diff --git a/src/os.c b/src/os.c index 0c3c9d898d..13c9abcab4 100644 --- a/src/os.c +++ b/src/os.c @@ -223,6 +223,7 @@ int sqlite3OsOpen( ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ + assert( zPath || (flags & SQLITE_OPEN_EXCLUSIVE) ); rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; diff --git a/src/pager.c b/src/pager.c index 95e6eb8af1..ca406b8c5d 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5786,6 +5786,7 @@ static int pager_open_journal(Pager *pPager){ if( pPager->tempFile ){ flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + flags |= SQLITE_OPEN_EXCLUSIVE; nSpill = sqlite3Config.nStmtSpill; }else{ flags |= SQLITE_OPEN_MAIN_JOURNAL; From 0f201fccc0b113e98641623a1ba7b25c42fbf346 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 16 Aug 2022 14:09:51 +0000 Subject: [PATCH 142/151] Omit an unnecessary test case from corruptN.test that does not work with the new RFC-7539 PRNG. FossilOrigin-Name: d9e8c65ed25c4f6222c737f4244d5362dbe433d357f7d133765157446cf4e925 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- test/corruptN.test | 15 +++++++++------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 28553e1e9b..376b7f52af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\sthe\sRC4-based\sPRNG\swith\sone\sbased\son\sChaCha20.\s\s3x\sfaster. -D 2022-08-16T00:04:40.434 +C Omit\san\sunnecessary\stest\scase\sfrom\scorruptN.test\sthat\sdoes\snot\swork\swith\sthe\nnew\sRFC-7539\sPRNG. +D 2022-08-16T14:09:51.629 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -854,7 +854,7 @@ F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 F test/corruptK.test 5b4212fe346699831c5ad559a62c54e11c0611bdde1ea8423a091f9c01aa32af F test/corruptL.test ecce40d7b9b909a670a42a45d86e30d927735d7e7f09041af438b19529d35532 F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067 -F test/corruptN.test 60b5a62944b4f0029ba07edaa5fd8e670539d6b0a8d99db26c068d435675cbfe +F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4 F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 F test/countofview.test e17d6e6688cf74f22783c9ec6e788c0790ee4fbbaee713affd00b1ac0bb39b86 @@ -1999,11 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c271096736889530f392ff37e631a77f3bc9c46b290dbda245fa05249f4410fc -R 169c772b3aaee1590ba8a07f447292d5 -T *branch * chacha20-prng -T *sym-chacha20-prng * -T -sym-trunk * +P 084d8776fa95c75440530028171c56547a341c9a952ba2f29bb533b538603c78 +R 4a1be98000c410a528d0befc32c8b6da U drh -Z 1e027b4a09350d19e88468e502211941 +Z af7ba7ff2c1330aa966ddca9675bb3a2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3e2548ba94..63acf48a4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -084d8776fa95c75440530028171c56547a341c9a952ba2f29bb533b538603c78 \ No newline at end of file +d9e8c65ed25c4f6222c737f4244d5362dbe433d357f7d133765157446cf4e925 \ No newline at end of file diff --git a/test/corruptN.test b/test/corruptN.test index 4685788223..376325f5c5 100644 --- a/test/corruptN.test +++ b/test/corruptN.test @@ -141,12 +141,15 @@ do_test 2.0 { | end c-b92b.txt.db }]} {} -prng_seed 0 db -do_catchsql_test 2.1 { -SELECT count(*) FROM sqlite_schema; -WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) -INSERT INTO t1(a) SELECT randomblob(null) FROM c; -} {1 {database disk image is malformed}} +# This test only works with the legacy RC4 PRNG +if 0 { + prng_seed 0 db + do_catchsql_test 2.1 { + SELECT count(*) FROM sqlite_schema; + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t1(a) SELECT randomblob(null) FROM c; + } {1 {database disk image is malformed}} +} reset_db if {![info exists ::G(perm:presql)]} { From 534945ad07002648e46a5f1db0cdeea216d79d79 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 16 Aug 2022 16:40:54 +0000 Subject: [PATCH 143/151] Fix an off-by-one error in the ChaCha20 initialization code. FossilOrigin-Name: 72e220eed446ea9a02a6ef03e09a01bcb8bbca1f3b32d2e0bf52a17d9722e2f0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/random.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 376b7f52af..33c8ab32a3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\san\sunnecessary\stest\scase\sfrom\scorruptN.test\sthat\sdoes\snot\swork\swith\sthe\nnew\sRFC-7539\sPRNG. -D 2022-08-16T14:09:51.629 +C Fix\san\soff-by-one\serror\sin\sthe\sChaCha20\sinitialization\scode. +D 2022-08-16T16:40:54.599 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -585,7 +585,7 @@ F src/pragma.c 6637d624c37a8909d3edfa9d7cf694d79b49d2a0827d8c52ef15dceb641783fa F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 -F src/random.c ff4b47317d22c9b6af581e25cb9243190082c57daaf175ae93c4adaa223a1e5e +F src/random.c af089dd588a3cbf85df14f9711c347f70050fcb73bc003376c8491065f6c317b F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 084d8776fa95c75440530028171c56547a341c9a952ba2f29bb533b538603c78 -R 4a1be98000c410a528d0befc32c8b6da +P d9e8c65ed25c4f6222c737f4244d5362dbe433d357f7d133765157446cf4e925 +R 6547827ce11f2be716204766ff5dff1b U drh -Z af7ba7ff2c1330aa966ddca9675bb3a2 +Z 947a70a0b2c6f8310490779060824f95 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 63acf48a4f..a9411d2af9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9e8c65ed25c4f6222c737f4244d5362dbe433d357f7d133765157446cf4e925 \ No newline at end of file +72e220eed446ea9a02a6ef03e09a01bcb8bbca1f3b32d2e0bf52a17d9722e2f0 \ No newline at end of file diff --git a/src/random.c b/src/random.c index 11c5a42b6d..8bf75156aa 100644 --- a/src/random.c +++ b/src/random.c @@ -108,7 +108,7 @@ void sqlite3_randomness(int N, void *pBuf){ }else{ sqlite3OsRandomness(pVfs, 44, (char*)&wsdPrng.s[4]); } - wsdPrng.s[16] = wsdPrng.s[12]; + wsdPrng.s[15] = wsdPrng.s[12]; wsdPrng.s[12] = 0; wsdPrng.n = 0; } From cd05aafa56ec8437c26528bde61be157806a4b3a Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 16 Aug 2022 16:57:33 +0000 Subject: [PATCH 144/151] Fix obsolete comments. Add new comments. Fix non-standard spacing. FossilOrigin-Name: a0d224c6a69941dad1f2b35edcc7ddee343b99eae2aeed74043461f3e97ef5b4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/random.c | 21 +++++++++------------ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 33c8ab32a3..db5ad92d5f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\serror\sin\sthe\sChaCha20\sinitialization\scode. -D 2022-08-16T16:40:54.599 +C Fix\sobsolete\scomments.\s\sAdd\snew\scomments.\s\sFix\snon-standard\sspacing. +D 2022-08-16T16:57:33.367 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -585,7 +585,7 @@ F src/pragma.c 6637d624c37a8909d3edfa9d7cf694d79b49d2a0827d8c52ef15dceb641783fa F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 -F src/random.c af089dd588a3cbf85df14f9711c347f70050fcb73bc003376c8491065f6c317b +F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9e8c65ed25c4f6222c737f4244d5362dbe433d357f7d133765157446cf4e925 -R 6547827ce11f2be716204766ff5dff1b +P 72e220eed446ea9a02a6ef03e09a01bcb8bbca1f3b32d2e0bf52a17d9722e2f0 +R cdb53b59d0af11c8f66b0b5bb02ede98 U drh -Z 947a70a0b2c6f8310490779060824f95 +Z 1158c2a25804bd20ee35e9fb8e722661 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a9411d2af9..54413fbc1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72e220eed446ea9a02a6ef03e09a01bcb8bbca1f3b32d2e0bf52a17d9722e2f0 \ No newline at end of file +a0d224c6a69941dad1f2b35edcc7ddee343b99eae2aeed74043461f3e97ef5b4 \ No newline at end of file diff --git a/src/random.c b/src/random.c index 8bf75156aa..9335fc49da 100644 --- a/src/random.c +++ b/src/random.c @@ -27,12 +27,15 @@ static SQLITE_WSD struct sqlite3PrngType { u8 n; /* Output bytes remaining */ } sqlite3Prng; + +/* The RFC-7539 ChaCha20 block function +*/ #define ROTL(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) -#define QR(a, b, c, d) ( \ - a += b, d ^= a, d = ROTL(d,16), \ - c += d, b ^= c, b = ROTL(b,12), \ - a += b, d ^= a, d = ROTL(d, 8), \ - c += d, b ^= c, b = ROTL(b, 7)) +#define QR(a, b, c, d) ( \ + a += b, d ^= a, d = ROTL(d,16), \ + c += d, b ^= c, b = ROTL(b,12), \ + a += b, d ^= a, d = ROTL(d, 8), \ + c += d, b ^= c, b = ROTL(b, 7)) static void chacha_block(u32 *out, const u32 *in){ int i; u32 x[16]; @@ -89,13 +92,7 @@ void sqlite3_randomness(int N, void *pBuf){ } /* Initialize the state of the random number generator once, - ** the first time this routine is called. The seed value does - ** not need to contain a lot of randomness since we are not - ** trying to do secure encryption or anything like that... - ** - ** Nothing in this file or anywhere else in SQLite does any kind of - ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random - ** number generator) not as an encryption device. + ** the first time this routine is called. */ if( wsdPrng.s[0]==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(0); From 0348fc722f83ebd2e0359b9a7388cd354a65f59c Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 17 Aug 2022 17:14:36 +0000 Subject: [PATCH 145/151] Minor code simplification in the code generator for PRAGMA integrity check. FossilOrigin-Name: a3b9c7a6c9be5c78a93e5125f16237d2d84b977eca81f527af0198e96435a995 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/pragma.c | 4 +--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f8f98704b7..761fdc8196 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\slegacy\sRC4-based\sPRNG\sto\suse\sthe\sRFC-7539\schacha20\salgorithm. -D 2022-08-16T17:18:00.622 +C Minor\scode\ssimplification\sin\sthe\scode\sgenerator\sfor\sPRAGMA\sintegrity\scheck. +D 2022-08-17T17:14:36.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -581,7 +581,7 @@ F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564 F src/pcache.c 084e638432c610f95aea72b8509f0845d2791293f39d1b82f0c0a7e089c3bb6b F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 0b4245cd4964e635f2630908c2533cd8e9da7af3ca592e23ae8730aa25ae5eb9 -F src/pragma.c 6637d624c37a8909d3edfa9d7cf694d79b49d2a0827d8c52ef15dceb641783fa +F src/pragma.c f857ee5dfdaa9ea00a04c37dbd33075d9d12b6f8c59cc8dec6de691d7bf08d3f F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 @@ -1999,9 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e123da49ccae61d591abded52f4721aa10f20d75935c9a3e3fe826a9b8df2317 a0d224c6a69941dad1f2b35edcc7ddee343b99eae2aeed74043461f3e97ef5b4 -R d14b7d3bd2befa6e359a3420d9b0b1d1 -T +closed a0d224c6a69941dad1f2b35edcc7ddee343b99eae2aeed74043461f3e97ef5b4 +P a0f801151925e882e120f6ab685dcacb9d3268d25b52bc665c5b927bcc7dda1e +R b8c3444adb3f1879cc53cf493efeffeb U drh -Z c066b8669f45d6d3a5c825d691d523b1 +Z a499216437f80b2eceb710f14f96178a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f073344af1..fc5f8a601c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0f801151925e882e120f6ab685dcacb9d3268d25b52bc665c5b927bcc7dda1e \ No newline at end of file +a3b9c7a6c9be5c78a93e5125f16237d2d84b977eca81f527af0198e96435a995 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 0a1bc37fae..37a73bdb62 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1796,9 +1796,7 @@ void sqlite3Pragma( } sqlite3VdbeJumpHere(v, jmp2); } - if( (pTab->tabFlags & TF_Strict)!=0 - && pCol->eCType!=COLTYPE_ANY - ){ + if( bStrict && pCol->eCType!=COLTYPE_ANY ){ jmp2 = sqlite3VdbeAddOp3(v, OP_IsNullOrType, 3, 0, sqlite3StdTypeMap[pCol->eCType-1]); VdbeCoverage(v); From 16b03c01965a6feae64ca5e202717f68df9415ad Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 17 Aug 2022 18:07:52 +0000 Subject: [PATCH 146/151] Enhance the "PRAGMA integrity_check" statement so that it verifies the rows of a WITHOUT ROWID table are in the correct order. FossilOrigin-Name: 62f934bff495850d0763e07ffa44a557f066ecba9d039363f32287213cba819f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 33 +++++++++++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 761fdc8196..68c53fa159 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scode\ssimplification\sin\sthe\scode\sgenerator\sfor\sPRAGMA\sintegrity\scheck. -D 2022-08-17T17:14:36.514 +C Enhance\sthe\s"PRAGMA\sintegrity_check"\sstatement\sso\sthat\sit\sverifies\sthe\srows\sof\na\sWITHOUT\sROWID\stable\sare\sin\sthe\scorrect\sorder. +D 2022-08-17T18:07:52.686 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -581,7 +581,7 @@ F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564 F src/pcache.c 084e638432c610f95aea72b8509f0845d2791293f39d1b82f0c0a7e089c3bb6b F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 0b4245cd4964e635f2630908c2533cd8e9da7af3ca592e23ae8730aa25ae5eb9 -F src/pragma.c f857ee5dfdaa9ea00a04c37dbd33075d9d12b6f8c59cc8dec6de691d7bf08d3f +F src/pragma.c b57a859a366472131194a9ad35cd76d5920577226b04c884b1b9085605faa280 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a0f801151925e882e120f6ab685dcacb9d3268d25b52bc665c5b927bcc7dda1e -R b8c3444adb3f1879cc53cf493efeffeb +P a3b9c7a6c9be5c78a93e5125f16237d2d84b977eca81f527af0198e96435a995 +R 3530f38cdde3fed1dbda570f4de01c0d U drh -Z a499216437f80b2eceb710f14f96178a +Z 5bcc55f0fe2746025d33da0258adb944 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fc5f8a601c..d75fa3c90f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a3b9c7a6c9be5c78a93e5125f16237d2d84b977eca81f527af0198e96435a995 \ No newline at end of file +62f934bff495850d0763e07ffa44a557f066ecba9d039363f32287213cba819f \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 37a73bdb62..2dd792f274 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1742,15 +1742,23 @@ void sqlite3Pragma( for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx, *pPk; - Index *pPrior = 0; + Index *pPrior = 0; /* Previous index */ int loopTop; int iDataCur, iIdxCur; int r1 = -1; int bStrict; + int r2; /* Previous key for WITHOUT ROWID tables */ if( !IsOrdinaryTable(pTab) ) continue; if( pObjTab && pObjTab!=pTab ) continue; - pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + if( isQuick || HasRowid(pTab) ){ + pPk = 0; + r2 = 0; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + r2 = sqlite3GetTempRange(pParse, pPk->nKeyCol); + sqlite3VdbeAddOp3(v, OP_Null, 1, r2, r2+pPk->nKeyCol-1); + } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); /* reg[7] counts the number of entries in the table. @@ -1769,6 +1777,24 @@ void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nNVCol-1,3); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); VdbeComment((v, "(right-most column)")); + if( pPk ){ + /* Verify WITHOUT ROWID keys are in ascending order */ + int a1; + char *zErr; + a1 = sqlite3VdbeAddOp4Int(v, OP_IdxGT, iDataCur, 0,r2,pPk->nKeyCol); + VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_IsNull, r2); VdbeCoverage(v); + zErr = sqlite3MPrintf(db, + "row not in PRIMARY KEY order for %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + sqlite3VdbeJumpHere(v, a1+1); + for(j=0; jnKeyCol; j++){ + sqlite3ExprCodeLoadIndexColumn(pParse, pPk, iDataCur, j, r2+j); + } + } } /* Verify that all NOT NULL columns really are NOT NULL. At the ** same time verify the type of the content of STRICT tables */ @@ -1895,6 +1921,9 @@ void sqlite3Pragma( integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, addr); } + if( pPk ){ + sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); + } } } } From 78ed74ef8e58e928215925f7709444d95239ebe8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 17 Aug 2022 20:18:34 +0000 Subject: [PATCH 147/151] In the ".dump" command of the CLI, if a schema statement ends with an unterminated comment, try to terminate that comment prior to appending the ";" at the end. [forum:/forumpost/d7be961c5c|Forum post d7be961c5c]. FossilOrigin-Name: 96e112da1ac56767cf49f26132833745b6020d2c60f5c36f86ca84f87ad30c81 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 68c53fa159..9c87b644f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\s"PRAGMA\sintegrity_check"\sstatement\sso\sthat\sit\sverifies\sthe\srows\sof\na\sWITHOUT\sROWID\stable\sare\sin\sthe\scorrect\sorder. -D 2022-08-17T18:07:52.686 +C In\sthe\s".dump"\scommand\sof\sthe\sCLI,\sif\sa\sschema\sstatement\sends\swith\san\nunterminated\scomment,\stry\sto\sterminate\sthat\scomment\sprior\sto\sappending\nthe\s";"\sat\sthe\send.\s\s[forum:/forumpost/d7be961c5c|Forum\spost\sd7be961c5c]. +D 2022-08-17T20:18:34.624 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -589,7 +589,7 @@ F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc -F src/shell.c.in 29749b34bbd19d0004fdb6f61f62659096e1c0b4dfb1ad2314e7fafbe9dd8d37 +F src/shell.c.in 269f682249c1bce2962883e5b99c8702b16a488a43b9ae186daa178713a93c5d F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a3b9c7a6c9be5c78a93e5125f16237d2d84b977eca81f527af0198e96435a995 -R 3530f38cdde3fed1dbda570f4de01c0d +P 62f934bff495850d0763e07ffa44a557f066ecba9d039363f32287213cba819f +R f81b87acda08a2789d4e7c27b7b89cff U drh -Z 5bcc55f0fe2746025d33da0258adb944 +Z 0ac82790ca7f9e87feb2c5a2f457751d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d75fa3c90f..0d1833b114 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62f934bff495850d0763e07ffa44a557f066ecba9d039363f32287213cba819f \ No newline at end of file +96e112da1ac56767cf49f26132833745b6020d2c60f5c36f86ca84f87ad30c81 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 641a93840d..5feadd7bf0 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1909,15 +1909,37 @@ static int shellAuth( ** ** This routine converts some CREATE TABLE statements for shadow tables ** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. +** +** If the schema statement in z[] contains a start-of-comment and if +** sqlite3_complete() returns false, try to terminate the comment before +** printing the result. https://sqlite.org/forum/forumpost/d7be961c5c */ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ + char *zToFree = 0; if( z==0 ) return; if( zTail==0 ) return; + if( zTail[0]==';' && (strstr(z, "/*")!=0 || strstr(z,"--")!=0) ){ + const char *zOrig = z; + static const char *azTerm[] = { "", "*/", "\n" }; + int i; + for(i=0; i Date: Fri, 19 Aug 2022 20:10:51 +0000 Subject: [PATCH 148/151] Add test case for fuzzer case crash-18fe4e. FossilOrigin-Name: 8372468bb5d8922cf20b8bbee34cfd6044ceb09c26a4efa79a5e6df2c7c4b730 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- test/dbpagefault.test | 23 ++++++++++++++++++++++- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9c87b644f1..385e177e73 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s".dump"\scommand\sof\sthe\sCLI,\sif\sa\sschema\sstatement\sends\swith\san\nunterminated\scomment,\stry\sto\sterminate\sthat\scomment\sprior\sto\sappending\nthe\s";"\sat\sthe\send.\s\s[forum:/forumpost/d7be961c5c|Forum\spost\sd7be961c5c]. -D 2022-08-17T20:18:34.624 +C Add\stest\scase\sfor\sfuzzer\scase\scrash-18fe4e. +D 2022-08-19T20:10:51.822 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -885,7 +885,7 @@ F test/dbfuzz001.test 55e1a3504f8dea84155e09912fe3b1c3ad77e0b1a938ec42ca03b8e51b F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee F test/dbfuzz2.c 4b3c12de4d98b1b2d908ab03d217d4619e47c8b23d5e67f8a6f2b1bdee7cae23 F test/dbpage.test fce29035c7566fd7835ec0f19422cb4b9c6944ce0e1b936ff8452443f92e887d -F test/dbpagefault.test e917cd250018b836e8d7d1a659c63bcd25737a8f31f15925b891b8f92247fdf9 +F test/dbpagefault.test 20fe3a2a295f1c8cb30195b7b58a010530ecbc060e53e146af9e3e4fde4cab15 F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed42759 F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef F test/decimal.test fcf403fd5585f47342234e153c4a4338cd737b8e0884ac66fc484df47dbcf1a7 @@ -1999,8 +1999,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 62f934bff495850d0763e07ffa44a557f066ecba9d039363f32287213cba819f -R f81b87acda08a2789d4e7c27b7b89cff -U drh -Z 0ac82790ca7f9e87feb2c5a2f457751d +P 96e112da1ac56767cf49f26132833745b6020d2c60f5c36f86ca84f87ad30c81 +R 28505063cdffb9084d2435033596eaeb +T *branch * crash-18fe4e +T *sym-crash-18fe4e * +T -sym-trunk * +U dan +Z 77ad7cb07d337524c18e3fbb9e8e199f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0d1833b114..ddf61111ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96e112da1ac56767cf49f26132833745b6020d2c60f5c36f86ca84f87ad30c81 \ No newline at end of file +8372468bb5d8922cf20b8bbee34cfd6044ceb09c26a4efa79a5e6df2c7c4b730 \ No newline at end of file diff --git a/test/dbpagefault.test b/test/dbpagefault.test index 550a567127..80cff00b49 100644 --- a/test/dbpagefault.test +++ b/test/dbpagefault.test @@ -22,7 +22,6 @@ if {[permutation] == "inmemory_journal"} { set testprefix dbpagefault - faultsim_save_and_close do_faultsim_test 1 -prep { faultsim_restore_and_reopen @@ -49,4 +48,26 @@ do_faultsim_test 2 -prep { faultsim_test_result {1 {no such schema}} {1 {SQL logic error}} } +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(z, b); + CREATE TRIGGER BEFORE INSERT ON x1 BEGIN + DELETE FROM sqlite_dbpage WHERE pgno=100; + UPDATE sqlite_dbpage SET data=null WHERE pgno=100; + END; +} + +do_faultsim_test 3 -prep { + catch { db close } + sqlite3 db test.db + execsql { PRAGMA trusted_schema = true } +} -body { + execsql { INSERT INTO x1 DEFAULT VALUES; } +} -test { + faultsim_test_result {0 {}} +} + + finish_test + + From 00378fde230fe91bdf3885b1e10d779ee96948c5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 20 Aug 2022 19:33:04 +0000 Subject: [PATCH 149/151] Enhance the enforcement of SQLITE_VTAB_DIRECTONLY so that it applies to DML statements within triggers. Do not allow DML stratements against virtual tables within triggers unless either the virtual table is SQLITE_VTAB_INNOCUOUS or PRAGMA trusted_schema is ON. FossilOrigin-Name: 9433ea4070f52135be64569057f439e7bdb4b3f425d87167c9ebda50011210c1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/delete.c | 38 ++++++++++++++++++++++++++++++++------ 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 9c87b644f1..fd6c9da95f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s".dump"\scommand\sof\sthe\sCLI,\sif\sa\sschema\sstatement\sends\swith\san\nunterminated\scomment,\stry\sto\sterminate\sthat\scomment\sprior\sto\sappending\nthe\s";"\sat\sthe\send.\s\s[forum:/forumpost/d7be961c5c|Forum\spost\sd7be961c5c]. -D 2022-08-17T20:18:34.624 +C Enhance\sthe\senforcement\sof\sSQLITE_VTAB_DIRECTONLY\sso\sthat\sit\sapplies\sto\nDML\sstatements\swithin\striggers.\s\sDo\snot\sallow\sDML\sstratements\sagainst\nvirtual\stables\swithin\striggers\sunless\seither\sthe\svirtual\stable\sis\nSQLITE_VTAB_INNOCUOUS\sor\sPRAGMA\strusted_schema\sis\sON. +D 2022-08-20T19:33:04.530 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -538,7 +538,7 @@ F src/ctime.c 93e4b5f4faf6d3f688988a116773259a4fbfb4ddac0e9bf9d0ae0429390c2543 F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d -F src/delete.c 13eca2beee5b758ed033a11230971310cc4a58fcd8f6bc33cad4f677c985e96c +F src/delete.c cba80ff7370f9d49e147470ef15f05cf00359f0725256a3703b422b6f376e270 F src/expr.c 0f72468b64eef40c280fd6b273c19f3b221444b00f535c7ba6f34faa657e63da F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 62f934bff495850d0763e07ffa44a557f066ecba9d039363f32287213cba819f -R f81b87acda08a2789d4e7c27b7b89cff +P 96e112da1ac56767cf49f26132833745b6020d2c60f5c36f86ca84f87ad30c81 +R b2aaecc820aeeb861481321e6e0d526f U drh -Z 0ac82790ca7f9e87feb2c5a2f457751d +Z 057552ff6703cfb91f8abf2e3faed7e1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0d1833b114..f007df01e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96e112da1ac56767cf49f26132833745b6020d2c60f5c36f86ca84f87ad30c81 \ No newline at end of file +9433ea4070f52135be64569057f439e7bdb4b3f425d87167c9ebda50011210c1 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 3d0e055be2..056d6f3bef 100644 --- a/src/delete.c +++ b/src/delete.c @@ -61,18 +61,42 @@ void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char *zColName){ ** 1) It is a virtual table and no implementation of the xUpdate method ** has been provided ** -** 2) It is a system table (i.e. sqlite_schema), this call is not +** 2) A trigger is currently being coded and the table is a virtual table +** that is SQLITE_VTAB_DIRECTONLY or if PRAGMA trusted_schema=OFF and +** the table is not SQLITE_VTAB_INNOCUOUS. +** +** 3) It is a system table (i.e. sqlite_schema), this call is not ** part of a nested parse and writable_schema pragma has not ** been specified ** -** 3) The table is a shadow table, the database connection is in +** 4) The table is a shadow table, the database connection is in ** defensive mode, and the current sqlite3_prepare() ** is for a top-level SQL statement. */ +static int vtabIsReadOnly(Parse *pParse, Table *pTab){ + if( sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ){ + return 1; + } + + /* Within triggers: + ** * Do not allow DELETE, INSERT, or UPDATE of SQLITE_VTAB_DIRECTONLY + ** virtual tables + ** * Only allow DELETE, INSERT, or UPDATE of non-SQLITE_VTAB_INNOCUOUS + ** virtual tables if PRAGMA trusted_schema=ON. + */ + if( pParse->pToplevel!=0 + && pTab->u.vtab.p->eVtabRisk > + ((pParse->db->flags & SQLITE_TrustedSchema)!=0) + ){ + sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"", + pTab->zName); + } + return 0; +} static int tabIsReadOnly(Parse *pParse, Table *pTab){ sqlite3 *db; if( IsVirtual(pTab) ){ - return sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0; + return vtabIsReadOnly(pParse, pTab); } if( (pTab->tabFlags & (TF_Readonly|TF_Shadow))==0 ) return 0; db = pParse->db; @@ -84,9 +108,11 @@ static int tabIsReadOnly(Parse *pParse, Table *pTab){ } /* -** Check to make sure the given table is writable. If it is not -** writable, generate an error message and return 1. If it is -** writable return 0; +** Check to make sure the given table is writable. +** +** If pTab is not writable -> generate an error message and return 1. +** If pTab is writable but other errors have occurred -> return 1. +** If pTab is writable and no prior errors -> return 0; */ int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ if( tabIsReadOnly(pParse, pTab) ){ From 41ce47c4f4bcae3882fdccec18a6100a85f4bba5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 22 Aug 2022 02:00:26 +0000 Subject: [PATCH 150/151] Add the new internal interface sqlite3DbNNFreeNN(db,ptr) where both the db and ptr parameters are guaranteed to be non-NULL. Use this where appropriate to save more than 2 million CPU cycles on the standard performance test. FossilOrigin-Name: e5eaa80e81fdf86f2875a912b880272b8d099b82b08e945a7988c5dd0fe9d6b5 --- manifest | 43 +++++++++++++++++++++---------------------- manifest.uuid | 2 +- src/btmutex.c | 1 + src/build.c | 25 +++++++++++++++---------- src/callback.c | 6 ++++-- src/delete.c | 2 +- src/expr.c | 11 +++++++---- src/fkey.c | 3 ++- src/insert.c | 2 +- src/malloc.c | 35 +++++++++++++++++++++++++++++++++++ src/prepare.c | 6 +++--- src/select.c | 6 ++++-- src/sqliteInt.h | 1 + src/tokenize.c | 2 +- src/vdbeaux.c | 34 ++++++++++++++++++++-------------- src/vtab.c | 3 ++- src/where.c | 11 +++++++---- src/wherecode.c | 4 ++-- 18 files changed, 128 insertions(+), 69 deletions(-) diff --git a/manifest b/manifest index 90ad483ca7..bfe61c9805 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\stest\scase\sfor\sdbsqlfuzz\s18fe4e257be7fa3ecfb0424ab7427e41e97ef9e3\nassertion\sfault,\swhich\sis\sfixed\swith\sthe\sprevious\scheck-in. -D 2022-08-20T19:45:41.430 +C Add\sthe\snew\sinternal\sinterface\ssqlite3DbNNFreeNN(db,ptr)\swhere\sboth\sthe\ndb\sand\sptr\sparameters\sare\sguaranteed\sto\sbe\snon-NULL.\s\sUse\sthis\swhere\nappropriate\sto\ssave\smore\sthan\s2\smillion\sCPU\scycles\son\sthe\sstandard\nperformance\stest. +D 2022-08-22T02:00:26.214 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -527,33 +527,33 @@ F src/attach.c 4431f82f0247bf3aaf91589acafdff77d1882235c95407b36da1585c765fbbc8 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d -F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 +F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca F src/btree.c 4a8d349b9ed4dc6d252c535227699d75319b633058a56432ebf43c9f56f9085e F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e -F src/build.c 1e96f35d5912a1606c9c9463dfd8eaffc76b2bc01207ee372cb3249eef5173c4 -F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c +F src/build.c 898884afd67d953808cb687babc15b66a10213f99fe2ce7db98960e959881f98 +F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 93e4b5f4faf6d3f688988a116773259a4fbfb4ddac0e9bf9d0ae0429390c2543 F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d -F src/delete.c cba80ff7370f9d49e147470ef15f05cf00359f0725256a3703b422b6f376e270 -F src/expr.c 0f72468b64eef40c280fd6b273c19f3b221444b00f535c7ba6f34faa657e63da +F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e +F src/expr.c 24e828db6b2fab8aabfb5d2c0d83dbdfc5a1972b1147fa893350e317ab7e282f F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c d965ede15d8360c09ed59348940649ee647b192e784466837d7aefa836d1d91e +F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 F src/global.c e83ee571b79ee3adc32e380cf554cf1254bc43763d23786c71721fbcdfbbb965 F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c a8c994e3c3b8b08f61745bedabdf5affc79584a2b3c80ee2e4f038817838bd0a +F src/insert.c aea5361767817f917b0f0f647a1f0b1621bd858938ae6ae545c3b6b9814b798f F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b F src/main.c b91c7e71af6f33640c35b8239a285040aad8dfcfdaaf979152e743c0f8017ea8 -F src/malloc.c 4a3785323104678a8b4b0a482fe0c2a80900e7468ddf76ab0f2ea1c79a8ca8cd +F src/malloc.c 24369414a0fa402451cec0f0d1e94b637ce8e93c93cd29038ffe7629a8780ebf F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 @@ -583,17 +583,17 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 0b4245cd4964e635f2630908c2533cd8e9da7af3ca592e23ae8730aa25ae5eb9 F src/pragma.c b57a859a366472131194a9ad35cd76d5920577226b04c884b1b9085605faa280 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 -F src/prepare.c c62820c15dcb63013519c8e41d9f928d7478672cc902cfd0581c733c271dbf45 +F src/prepare.c 971d5819a4bda88038c2283d71fc0a2974ffc3dd480f9bd941341017abacfd1b F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 4750fbe9d8ecb7236baf7a9bea4299bb87126e08c209645666a0ae8f0efbe0fc +F src/select.c ccce37e7fbe71089cf6aec91e7134c9c0c1d4840cff9f02587bbc71240d914a5 F src/shell.c.in 269f682249c1bce2962883e5b99c8702b16a488a43b9ae186daa178713a93c5d F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d -F src/sqliteInt.h 3ae1d20f579149c18ddd995bbeffabf036cad9f4359dc2f27dc1b778d108ff35 +F src/sqliteInt.h 51ccf8da51d5263a21543631f0c3bdca06e9fbe8238a1b2c8d7ba4f4cd118aea F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4a3da6d77eeb3531cb0dbdf7047772a2a1b99f98c69e90ce009c75fe6328b2c0 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -652,7 +652,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 36eb0799e487759bbe73e5742b82ee676f06cea2515ff578d03c59a74ccf2d6c +F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 F src/trigger.c 61bea163b1fa3039bc572ed8312461b978e5c527e5301f302b078f4c1ccdec6a F src/update.c c52a7991bece0453d22c77c08469512ee2f1391c12503fd347d1c939220c5877 @@ -664,20 +664,20 @@ F src/vdbe.c 4e57ac969bb2252598024ee3ebcb0885cb7976f1606e2bd77975a506da93ed50 F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da F src/vdbeapi.c 4cfbf7ec3ed60366a38655f3f10316c5a3d68f6d4d06e462f88679392611c756 -F src/vdbeaux.c 8584f4a20997fd918f0d957ab4f73d7411159772297c5020c188973bdc41dbf0 +F src/vdbeaux.c cd9fc2fb24755366b2718f5e9e02103c39804bdc9e630620aaec08bb3093c46c F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c c3ce80af15e2ff5c2824a8db881681cbf511376f13613da020bac6d320c535b1 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c -F src/vtab.c 3d72c780d1ea08906a198e4f033921a658a54590e3ed72c544995d84f3f9464a +F src/vtab.c bb53f9e2eaeecca07158643dd3d5039cf13b525fe2d267e113b39a36f374556c F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 8bfe41220558a03be393d064ef3d942cd1ceaf57cd88a85b6cb59b7dda555202 +F src/where.c 6d1ee6f6bc048406554176eb6146b27ac16ec96cca018848aa0b9836b2b71143 F src/whereInt.h b48ca529ffe293c18cbfa8326af18a09e39910de66fb3e96ef788c7cbf8ef3a7 -F src/wherecode.c 719a5bb0102711a6ebed8d4385b831a27ce679172f3ae8e9aaec0dc1e415a95a +F src/wherecode.c 6bb1cf9d0a4e3e04dab0bf0ea4a8d936a0dcc05a7e2207beeda6c61aea6dd341 F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 F src/window.c 928e215840e2f2d9a2746e018c9643ef42c66c4ab6630ef0df7fa388fa145e86 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1999,9 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9433ea4070f52135be64569057f439e7bdb4b3f425d87167c9ebda50011210c1 8372468bb5d8922cf20b8bbee34cfd6044ceb09c26a4efa79a5e6df2c7c4b730 -R 8a2bc608dd9e0c10d3cc034d5ec85f92 -T +closed 8372468bb5d8922cf20b8bbee34cfd6044ceb09c26a4efa79a5e6df2c7c4b730 +P 2d13ec086e96a5446462ce0f689c40c8196e740cd693f5967bfe9eb961f03463 +R bf88e8a14ba34da9f9b58cc1ee676092 U drh -Z 96d7e346000d7bc7e6cdccdd39e9e0b9 +Z 1f007ec3ec70bab2436c890728aeb636 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5136d0d2e9..1388f09e3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d13ec086e96a5446462ce0f689c40c8196e740cd693f5967bfe9eb961f03463 \ No newline at end of file +e5eaa80e81fdf86f2875a912b880272b8d099b82b08e945a7988c5dd0fe9d6b5 \ No newline at end of file diff --git a/src/btmutex.c b/src/btmutex.c index 275a93ff21..45745b001d 100644 --- a/src/btmutex.c +++ b/src/btmutex.c @@ -252,6 +252,7 @@ int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){ int sqlite3SchemaMutexHeld(sqlite3 *db, int iDb, Schema *pSchema){ Btree *p; assert( db!=0 ); + if( db->pVfs==0 && db->nDb==0 ) return 1; if( pSchema ) iDb = sqlite3SchemaToIndex(db, pSchema); assert( iDb>=0 && iDbnDb ); if( !sqlite3_mutex_held(db->mutex) ) return 0; diff --git a/src/build.c b/src/build.c index e3e79df05d..094cf7d61b 100644 --- a/src/build.c +++ b/src/build.c @@ -775,16 +775,17 @@ void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); + assert( db!=0 ); if( (pCol = pTable->aCol)!=0 ){ for(i=0; inCol; i++, pCol++){ assert( pCol->zCnName==0 || pCol->hName==sqlite3StrIHash(pCol->zCnName) ); sqlite3DbFree(db, pCol->zCnName); } - sqlite3DbFree(db, pTable->aCol); + sqlite3DbNNFreeNN(db, pTable->aCol); if( IsOrdinaryTable(pTable) ){ sqlite3ExprListDelete(db, pTable->u.tab.pDfltList); } - if( db==0 || db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ pTable->aCol = 0; pTable->nCol = 0; if( IsOrdinaryTable(pTable) ){ @@ -821,7 +822,8 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ ** a Table object that was going to be marked ephemeral. So do not check ** that no lookaside memory is used in this case either. */ int nLookaside = 0; - if( db && !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ + assert( db!=0 ); + if( !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ nLookaside = sqlite3LookasideUsed(db, 0); } #endif @@ -831,7 +833,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ pNext = pIndex->pNext; assert( pIndex->pSchema==pTable->pSchema || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); - if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){ + if( db->pnBytesFreed==0 && !IsVirtual(pTable) ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( &pIndex->pSchema->idxHash, zName, 0 @@ -868,8 +870,9 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ } void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Do not delete the table until the reference count reaches zero. */ + assert( db!=0 ); if( !pTable ) return; - if( ((!db || db->pnBytesFreed==0) && (--pTable->nTabRef)>0) ) return; + if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return; deleteTable(db, pTable); } @@ -4689,12 +4692,13 @@ IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){ */ void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ int i; + assert( db!=0 ); if( pList==0 ) return; assert( pList->eU4!=EU4_EXPR ); /* EU4_EXPR mode is not currently used */ for(i=0; inId; i++){ sqlite3DbFree(db, pList->a[i].zName); } - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } /* @@ -4897,11 +4901,12 @@ void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ int i; SrcItem *pItem; + assert( db!=0 ); if( pList==0 ) return; for(pItem=pList->a, i=0; inSrc; i++, pItem++){ - if( pItem->zDatabase ) sqlite3DbFreeNN(db, pItem->zDatabase); - sqlite3DbFree(db, pItem->zName); - if( pItem->zAlias ) sqlite3DbFreeNN(db, pItem->zAlias); + if( pItem->zDatabase ) sqlite3DbNNFreeNN(db, pItem->zDatabase); + if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); + if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pTab); @@ -4912,7 +4917,7 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3ExprDelete(db, pItem->u3.pOn); } } - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } /* diff --git a/src/callback.c b/src/callback.c index 7d8f9dcbce..6cbe8e5847 100644 --- a/src/callback.c +++ b/src/callback.c @@ -490,19 +490,21 @@ void sqlite3SchemaClear(void *p){ Hash temp2; HashElem *pElem; Schema *pSchema = (Schema *)p; + sqlite3 xdb; + memset(&xdb, 0, sizeof(xdb)); temp1 = pSchema->tblHash; temp2 = pSchema->trigHash; sqlite3HashInit(&pSchema->trigHash); sqlite3HashClear(&pSchema->idxHash); for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ - sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); + sqlite3DeleteTrigger(&xdb, (Trigger*)sqliteHashData(pElem)); } sqlite3HashClear(&temp2); sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); - sqlite3DeleteTable(0, pTab); + sqlite3DeleteTable(&xdb, pTab); } sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); diff --git a/src/delete.c b/src/delete.c index 056d6f3bef..704a3c7110 100644 --- a/src/delete.c +++ b/src/delete.c @@ -676,7 +676,7 @@ delete_from_cleanup: sqlite3ExprListDelete(db, pOrderBy); sqlite3ExprDelete(db, pLimit); #endif - sqlite3DbFree(db, aToOpen); + if( aToOpen ) sqlite3DbNNFreeNN(db, aToOpen); return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise diff --git a/src/expr.c b/src/expr.c index be3e73294f..8572512926 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1220,6 +1220,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ */ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); + assert( db!=0 ); assert( !ExprUseUValue(p) || p->u.iValue>=0 ); assert( !ExprUseYWin(p) || !ExprUseYSub(p) ); assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed ); @@ -1252,7 +1253,7 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ } } if( !ExprHasProperty(p, EP_Static) ){ - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } } void sqlite3ExprDelete(sqlite3 *db, Expr *p){ @@ -2038,12 +2039,13 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ int i = pList->nExpr; struct ExprList_item *pItem = pList->a; assert( pList->nExpr>0 ); + assert( db!=0 ); do{ sqlite3ExprDelete(db, pItem->pExpr); - sqlite3DbFree(db, pItem->zEName); + if( pItem->zEName ) sqlite3DbNNFreeNN(db, pItem->zEName); pItem++; }while( --i>0 ); - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ if( pList ) exprListDeleteNN(db, pList); @@ -6075,6 +6077,7 @@ static int exprRefToSrcList(Walker *pWalker, Expr *pExpr){ int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList *pSrcList){ Walker w; struct RefSrcList x; + assert( pParse->db!=0 ); memset(&w, 0, sizeof(w)); memset(&x, 0, sizeof(x)); w.xExprCallback = exprRefToSrcList; @@ -6091,7 +6094,7 @@ int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList *pSrcList){ sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter); } #endif - sqlite3DbFree(pParse->db, x.aiExclude); + if( x.aiExclude ) sqlite3DbNNFreeNN(pParse->db, x.aiExclude); if( w.eCode & 0x01 ){ return 1; }else if( w.eCode ){ diff --git a/src/fkey.c b/src/fkey.c index a057925145..cae6bb19d3 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -1439,11 +1439,12 @@ void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pNext; /* Copy of pFKey->pNextFrom */ assert( IsOrdinaryTable(pTab) ); + assert( db!=0 ); for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pNext){ assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); /* Remove the FK from the fkeyHash hash table. */ - if( !db || db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ if( pFKey->pPrevTo ){ pFKey->pPrevTo->pNextTo = pFKey->pNextTo; }else{ diff --git a/src/insert.c b/src/insert.c index 6de304848e..6c71391a1b 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1417,7 +1417,7 @@ insert_cleanup: sqlite3UpsertDelete(db, pUpsert); sqlite3SelectDelete(db, pSelect); sqlite3IdListDelete(db, pColumn); - sqlite3DbFree(db, aRegIdx); + if( aRegIdx ) sqlite3DbNNFreeNN(db, aRegIdx); } /* Make sure "isView" and other macros defined above are undefined. Otherwise diff --git a/src/malloc.c b/src/malloc.c index c508bf752c..d0ed48dba1 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -441,6 +441,41 @@ void sqlite3DbFreeNN(sqlite3 *db, void *p){ sqlite3MemdebugSetType(p, MEMTYPE_HEAP); sqlite3_free(p); } +void sqlite3DbNNFreeNN(sqlite3 *db, void *p){ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( p!=0 ); + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; +#ifdef SQLITE_DEBUG + memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = pBuf; + return; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; +#ifdef SQLITE_DEBUG + memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + return; + } + } + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); +} void sqlite3DbFree(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); if( p ) sqlite3DbFreeNN(db, p); diff --git a/src/prepare.c b/src/prepare.c index f55ff72fcb..f66c366b99 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -574,15 +574,15 @@ void sqlite3ParseObjectReset(Parse *pParse){ assert( db->pParse==pParse ); assert( pParse->nested==0 ); #ifndef SQLITE_OMIT_SHARED_CACHE - sqlite3DbFree(db, pParse->aTableLock); + if( pParse->aTableLock ) sqlite3DbNNFreeNN(db, pParse->aTableLock); #endif while( pParse->pCleanup ){ ParseCleanup *pCleanup = pParse->pCleanup; pParse->pCleanup = pCleanup->pNext; pCleanup->xCleanup(db, pCleanup->pPtr); - sqlite3DbFreeNN(db, pCleanup); + sqlite3DbNNFreeNN(db, pCleanup); } - sqlite3DbFree(db, pParse->aLabel); + if( pParse->aLabel ) sqlite3DbNNFreeNN(db, pParse->aLabel); if( pParse->pConstExpr ){ sqlite3ExprListDelete(db, pParse->pConstExpr); } diff --git a/src/select.c b/src/select.c index 3f80e44ce2..39ec70f2b5 100644 --- a/src/select.c +++ b/src/select.c @@ -76,6 +76,7 @@ struct SortCtx { ** If bFree==0, Leave the first Select object unfreed */ static void clearSelect(sqlite3 *db, Select *p, int bFree){ + assert( db!=0 ); while( p ){ Select *pPrior = p->pPrior; sqlite3ExprListDelete(db, p->pEList); @@ -95,7 +96,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3WindowUnlinkFromSelect(p->pWin); } #endif - if( bFree ) sqlite3DbFreeNN(db, p); + if( bFree ) sqlite3DbNNFreeNN(db, p); p = pPrior; bFree = 1; } @@ -1501,9 +1502,10 @@ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ */ void sqlite3KeyInfoUnref(KeyInfo *p){ if( p ){ + assert( p->db!=0 ); assert( p->nRef>0 ); p->nRef--; - if( p->nRef==0 ) sqlite3DbFreeNN(p->db, p); + if( p->nRef==0 ) sqlite3DbNNFreeNN(p->db, p); } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 32e0e78e83..c53b4f3bb9 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4364,6 +4364,7 @@ void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64); void *sqlite3DbRealloc(sqlite3 *, void *, u64); void sqlite3DbFree(sqlite3*, void*); void sqlite3DbFreeNN(sqlite3*, void*); +void sqlite3DbNNFreeNN(sqlite3*, void*); int sqlite3MallocSize(const void*); int sqlite3DbMallocSize(sqlite3*, const void*); void *sqlite3PageMalloc(int); diff --git a/src/tokenize.c b/src/tokenize.c index b147cdb40f..8dac9ece52 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -711,7 +711,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ if( pParse->pNewTrigger && !IN_RENAME_OBJECT ){ sqlite3DeleteTrigger(db, pParse->pNewTrigger); } - if( pParse->pVList ) sqlite3DbFreeNN(db, pParse->pVList); + if( pParse->pVList ) sqlite3DbNNFreeNN(db, pParse->pVList); db->pParse = pParentParse; assert( nErr==0 || pParse->rc!=SQLITE_OK ); return nErr; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 8ab7978fa1..ca711d8d0d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -884,7 +884,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ } resolve_p2_values_loop_exit: if( aLabel ){ - sqlite3DbFreeNN(p->db, pParse->aLabel); + sqlite3DbNNFreeNN(p->db, pParse->aLabel); pParse->aLabel = 0; } pParse->nLabel = 0; @@ -1195,8 +1195,9 @@ void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ ** the FuncDef is not ephermal, then do nothing. */ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ + assert( db!=0 ); if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ - sqlite3DbFreeNN(db, pDef); + sqlite3DbNNFreeNN(db, pDef); } } @@ -1205,11 +1206,12 @@ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ */ static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){ if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){ + assert( db!=0 ); freeEphemeralFunction(db, p->pFunc); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } static void freeP4(sqlite3 *db, int p4type, void *p4){ assert( db ); @@ -1222,7 +1224,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ case P4_INT64: case P4_DYNAMIC: case P4_INTARRAY: { - sqlite3DbFree(db, p4); + if( p4 ) sqlite3DbNNFreeNN(db, p4); break; } case P4_KEYINFO: { @@ -1261,6 +1263,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ */ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ assert( nOp>=0 ); + assert( db!=0 ); if( aOp ){ Op *pOp = &aOp[nOp-1]; while(1){ /* Exit via break */ @@ -1271,7 +1274,7 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( pOp==aOp ) break; pOp--; } - sqlite3DbFreeNN(db, aOp); + sqlite3DbNNFreeNN(db, aOp); } } @@ -2025,7 +2028,7 @@ static void releaseMemArray(Mem *p, int N){ sqlite3VdbeMemRelease(p); p->flags = MEM_Undefined; }else if( p->szMalloc ){ - sqlite3DbFreeNN(db, p->zMalloc); + sqlite3DbNNFreeNN(db, p->zMalloc); p->szMalloc = 0; p->flags = MEM_Undefined; } @@ -3546,10 +3549,11 @@ void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, int mask){ */ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; + assert( db!=0 ); assert( p->db==0 || p->db==db ); if( p->aColName ){ releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); - sqlite3DbFreeNN(db, p->aColName); + sqlite3DbNNFreeNN(db, p->aColName); } for(pSub=p->pProgram; pSub; pSub=pNext){ pNext = pSub->pNext; @@ -3558,11 +3562,11 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ } if( p->eVdbeState!=VDBE_INIT_STATE ){ releaseMemArray(p->aVar, p->nVar); - if( p->pVList ) sqlite3DbFreeNN(db, p->pVList); - if( p->pFree ) sqlite3DbFreeNN(db, p->pFree); + if( p->pVList ) sqlite3DbNNFreeNN(db, p->pVList); + if( p->pFree ) sqlite3DbNNFreeNN(db, p->pFree); } vdbeFreeOpArray(db, p->aOp, p->nOp); - sqlite3DbFree(db, p->zSql); + if( p->zSql ) sqlite3DbNNFreeNN(db, p->zSql); #ifdef SQLITE_ENABLE_NORMALIZE sqlite3DbFree(db, p->zNormSql); { @@ -3592,6 +3596,7 @@ void sqlite3VdbeDelete(Vdbe *p){ assert( p!=0 ); db = p->db; + assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); sqlite3VdbeClearObject(db, p); if( db->pnBytesFreed==0 ){ @@ -3605,7 +3610,7 @@ void sqlite3VdbeDelete(Vdbe *p){ p->pNext->pPrev = p->pPrev; } } - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } /* @@ -5221,13 +5226,14 @@ void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ ** the vdbeUnpackRecord() function found in vdbeapi.c. */ static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){ + assert( db!=0 ); if( p ){ int i; for(i=0; iaMem[i]; if( pMem->zMalloc ) sqlite3VdbeMemReleaseMalloc(pMem); } - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -5298,7 +5304,7 @@ void sqlite3VdbePreUpdateHook( for(i=0; inField; i++){ sqlite3VdbeMemRelease(&preupdate.aNew[i]); } - sqlite3DbFreeNN(db, preupdate.aNew); + sqlite3DbNNFreeNN(db, preupdate.aNew); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ diff --git a/src/vtab.c b/src/vtab.c index b50ccd24a6..9bf031faf0 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -340,7 +340,8 @@ void sqlite3VtabUnlockList(sqlite3 *db){ */ void sqlite3VtabClear(sqlite3 *db, Table *p){ assert( IsVirtual(p) ); - if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); + assert( db!=0 ); + if( db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); if( p->u.vtab.azArg ){ int i; for(i=0; iu.vtab.nArg; i++){ diff --git a/src/where.c b/src/where.c index 9c4678345e..0a6f5e6273 100644 --- a/src/where.c +++ b/src/where.c @@ -2251,8 +2251,9 @@ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ ** Delete a WhereLoop object */ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + assert( db!=0 ); whereLoopClear(db, p); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } /* @@ -2260,6 +2261,7 @@ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ assert( pWInfo!=0 ); + assert( db!=0 ); sqlite3WhereClauseClear(&pWInfo->sWC); while( pWInfo->pLoops ){ WhereLoop *p = pWInfo->pLoops; @@ -2269,10 +2271,10 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ assert( pWInfo->pExprMods==0 ); while( pWInfo->pMemToFree ){ WhereMemBlock *pNext = pWInfo->pMemToFree->pNext; - sqlite3DbFreeNN(db, pWInfo->pMemToFree); + sqlite3DbNNFreeNN(db, pWInfo->pMemToFree); pWInfo->pMemToFree = pNext; } - sqlite3DbFreeNN(db, pWInfo); + sqlite3DbNNFreeNN(db, pWInfo); } /* Undo all Expr node modifications @@ -5073,7 +5075,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ - sqlite3DbFreeNN(db, pSpace); + assert( db!=0 ); + sqlite3DbNNFreeNN(db, pSpace); return SQLITE_OK; } diff --git a/src/wherecode.c b/src/wherecode.c index 85df64279c..4c34ea0dc6 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -2109,8 +2109,8 @@ Bitmask sqlite3WhereCodeOneLoopStart( } nConstraint++; } - sqlite3DbFree(db, zStartAff); - sqlite3DbFree(db, zEndAff); + if( zStartAff ) sqlite3DbNNFreeNN(db, zStartAff); + if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff); /* Top of the loop body */ if( pLevel->p2==0 ) pLevel->p2 = sqlite3VdbeCurrentAddr(v); From 376860ba88f75862e9110bf5858ff75a4e17b17c Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 22 Aug 2022 15:18:37 +0000 Subject: [PATCH 151/151] Performance enhancement for sqlite3DbFree(). FossilOrigin-Name: c296a9d958ec360fc8d217363348b4918d665bccb3c4f27503a2dcef7db49052 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/main.c | 7 ++++--- src/malloc.c | 38 ++++++++++++++++++++------------------ src/sqliteInt.h | 1 + src/status.c | 6 ++++++ src/vdbeapi.c | 3 +++ 7 files changed, 45 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index bfe61c9805..56bf1295a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\snew\sinternal\sinterface\ssqlite3DbNNFreeNN(db,ptr)\swhere\sboth\sthe\ndb\sand\sptr\sparameters\sare\sguaranteed\sto\sbe\snon-NULL.\s\sUse\sthis\swhere\nappropriate\sto\ssave\smore\sthan\s2\smillion\sCPU\scycles\son\sthe\sstandard\nperformance\stest. -D 2022-08-22T02:00:26.214 +C Performance\senhancement\sfor\ssqlite3DbFree(). +D 2022-08-22T15:18:37.415 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -552,8 +552,8 @@ F src/insert.c aea5361767817f917b0f0f647a1f0b1621bd858938ae6ae545c3b6b9814b798f F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b -F src/main.c b91c7e71af6f33640c35b8239a285040aad8dfcfdaaf979152e743c0f8017ea8 -F src/malloc.c 24369414a0fa402451cec0f0d1e94b637ce8e93c93cd29038ffe7629a8780ebf +F src/main.c 211f5fd4110b0565fc2135d60ed12b4ef8a29d634414f471b5f957aa7b7f2afa +F src/malloc.c b7a3430cbe91d3e8e04fc10c2041b3a19794e63556ad2441a13d8dadd9b2bafc F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 @@ -593,9 +593,9 @@ F src/shell.c.in 269f682249c1bce2962883e5b99c8702b16a488a43b9ae186daa178713a93c5 F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d -F src/sqliteInt.h 51ccf8da51d5263a21543631f0c3bdca06e9fbe8238a1b2c8d7ba4f4cd118aea +F src/sqliteInt.h ebf18764e404a2cef39ae5bfc8dd4a83bf0d70be1c444a4fbd8539eb35ef6ffd F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 -F src/status.c 4a3da6d77eeb3531cb0dbdf7047772a2a1b99f98c69e90ce009c75fe6328b2c0 +F src/status.c 1593e408e8e9ac956e8ecdf7db2728d8aa3e9da3405849f08c0f1f7ee7df86b5 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 4e64ba300a5a26e0f1170e09032429faeb65e45e8f3d1a7833e8edb69fc2979e F src/test1.c 1356984e97bff07e4a8cc3863e892f05b3348678a74783bb6f350b76316736f1 @@ -663,7 +663,7 @@ F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 F src/vdbe.c 4e57ac969bb2252598024ee3ebcb0885cb7976f1606e2bd77975a506da93ed50 F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 2cad0aeeb106371ed0e0946bab89f60627087068847afc2451c05056961c18da -F src/vdbeapi.c 4cfbf7ec3ed60366a38655f3f10316c5a3d68f6d4d06e462f88679392611c756 +F src/vdbeapi.c 6af7b32ada7ef8facafa91f57aedabd91633f254b34e9c0901ab402d4238e553 F src/vdbeaux.c cd9fc2fb24755366b2718f5e9e02103c39804bdc9e630620aaec08bb3093c46c F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c c3ce80af15e2ff5c2824a8db881681cbf511376f13613da020bac6d320c535b1 @@ -1999,8 +1999,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2d13ec086e96a5446462ce0f689c40c8196e740cd693f5967bfe9eb961f03463 -R bf88e8a14ba34da9f9b58cc1ee676092 +P e5eaa80e81fdf86f2875a912b880272b8d099b82b08e945a7988c5dd0fe9d6b5 +R 5d3ac911bfb51b81f7438edbe16f7f7d U drh -Z 1f007ec3ec70bab2436c890728aeb636 +Z 2b49dc0de08069abebcad82ed9e5f9e0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1388f09e3f..e6a20a4f19 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e5eaa80e81fdf86f2875a912b880272b8d099b82b08e945a7988c5dd0fe9d6b5 \ No newline at end of file +c296a9d958ec360fc8d217363348b4918d665bccb3c4f27503a2dcef7db49052 \ No newline at end of file diff --git a/src/main.c b/src/main.c index d905ac8ead..d982c3dfb7 100644 --- a/src/main.c +++ b/src/main.c @@ -824,18 +824,19 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ db->lookaside.bMalloced = pBuf==0 ?1:0; db->lookaside.nSlot = nBig+nSm; }else{ - db->lookaside.pStart = db; + db->lookaside.pStart = 0; #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE db->lookaside.pSmallInit = 0; db->lookaside.pSmallFree = 0; - db->lookaside.pMiddle = db; + db->lookaside.pMiddle = 0; #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ - db->lookaside.pEnd = db; + db->lookaside.pEnd = 0; db->lookaside.bDisable = 1; db->lookaside.sz = 0; db->lookaside.bMalloced = 0; db->lookaside.nSlot = 0; } + db->lookaside.pTrueEnd = db->lookaside.pEnd; assert( sqlite3LookasideUsed(db,0)==0 ); #endif /* SQLITE_OMIT_LOOKASIDE */ return SQLITE_OK; diff --git a/src/malloc.c b/src/malloc.c index d0ed48dba1..3fcedf8a37 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -317,7 +317,7 @@ void *sqlite3_malloc64(sqlite3_uint64 n){ */ #ifndef SQLITE_OMIT_LOOKASIDE static int isLookaside(sqlite3 *db, const void *p){ - return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd); + return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pTrueEnd); } #else #define isLookaside(A,B) 0 @@ -341,18 +341,16 @@ static int lookasideMallocSize(sqlite3 *db, const void *p){ int sqlite3DbMallocSize(sqlite3 *db, const void *p){ assert( p!=0 ); #ifdef SQLITE_DEBUG - if( db==0 || !isLookaside(db,p) ){ - if( db==0 ){ - assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - }else{ - assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - } + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + }else if( !isLookaside(db,p) ){ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); } #endif if( db ){ - if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ + if( ((uptr)p)<(uptr)(db->lookaside.pTrueEnd) ){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ assert( sqlite3_mutex_held(db->mutex) ); @@ -408,14 +406,11 @@ void sqlite3DbFreeNN(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); assert( p!=0 ); if( db ){ - if( db->pnBytesFreed ){ - measureAllocationSize(db, p); - return; - } if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); #ifdef SQLITE_DEBUG memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ #endif @@ -426,6 +421,7 @@ void sqlite3DbFreeNN(sqlite3 *db, void *p){ #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); #ifdef SQLITE_DEBUG memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ #endif @@ -434,6 +430,10 @@ void sqlite3DbFreeNN(sqlite3 *db, void *p){ return; } } + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); @@ -445,14 +445,11 @@ void sqlite3DbNNFreeNN(sqlite3 *db, void *p){ assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); assert( p!=0 ); - if( db->pnBytesFreed ){ - measureAllocationSize(db, p); - return; - } if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); #ifdef SQLITE_DEBUG memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ #endif @@ -463,6 +460,7 @@ void sqlite3DbNNFreeNN(sqlite3 *db, void *p){ #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); #ifdef SQLITE_DEBUG memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ #endif @@ -471,6 +469,10 @@ void sqlite3DbNNFreeNN(sqlite3 *db, void *p){ return; } } + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c53b4f3bb9..21d15378f1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1438,6 +1438,7 @@ struct Lookaside { #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ void *pStart; /* First byte of available memory space */ void *pEnd; /* First byte past end of available space */ + void *pTrueEnd; /* True value of pEnd, when db->pnBytesFreed!=0 */ }; struct LookasideSlot { LookasideSlot *pNext; /* Next buffer in the list of free buffers */ diff --git a/src/status.c b/src/status.c index e7f2158446..909aefe762 100644 --- a/src/status.c +++ b/src/status.c @@ -291,6 +291,8 @@ int sqlite3_db_status( sqlite3BtreeEnterAll(db); db->pnBytesFreed = &nByte; + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; for(i=0; inDb; i++){ Schema *pSchema = db->aDb[i].pSchema; if( ALWAYS(pSchema!=0) ){ @@ -316,6 +318,7 @@ int sqlite3_db_status( } } db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3BtreeLeaveAll(db); *pHighwater = 0; @@ -333,9 +336,12 @@ int sqlite3_db_status( int nByte = 0; /* Used to accumulate return value */ db->pnBytesFreed = &nByte; + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ sqlite3VdbeDelete(pVdbe); } + db->lookaside.pEnd = db->lookaside.pTrueEnd; db->pnBytesFreed = 0; *pHighwater = 0; /* IMP: R-64479-57858 */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 5d3e5601e5..2be9f7992c 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1836,8 +1836,11 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ sqlite3_mutex_enter(db->mutex); v = 0; db->pnBytesFreed = (int*)&v; + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; sqlite3VdbeDelete(pVdbe); db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3_mutex_leave(db->mutex); }else{ v = pVdbe->aCounter[op];