From 0546a284fa55856f0496a7f2d07f452e3d45245c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Dec 2023 16:21:22 +0000 Subject: [PATCH 01/24] Enhance the (undocumented, debug-only) json_parse() SQL function so that it returns the text rendering of the JSONB parse of the input, rather than printing the rendering on stdout. FossilOrigin-Name: 056de8d551dcbdf1d162e2db15ed418fa9c786f900cd3972ef8a1dea3f4f3aa1 --- manifest | 14 +++++----- manifest.uuid | 2 +- src/json.c | 76 +++++++++++++++++++++++++++++++-------------------- 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 692bc45510..0aaf21b3f9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\sshell\stool\s(not\slibrary)\scausing\san\sout-of-bounds\swrite\sif\san\s".open"\scommand\sfailed,\sthen\sthe\suser\spressed\sctrl-c\sto\sinterrupt\sa\squery\srunning\son\sthe\ssubstitute\sin-memory\sdatabase. -D 2023-12-27T16:24:53.514 +C Enhance\sthe\s(undocumented,\sdebug-only)\sjson_parse()\sSQL\sfunction\sso\sthat\sit\nreturns\sthe\stext\srendering\sof\sthe\sJSONB\sparse\sof\sthe\sinput,\srather\sthan\sprinting\nthe\srendering\son\sstdout. +D 2023-12-28T16:21:22.018 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -697,7 +697,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c bc90605da937ca0cd72ff0492216fbb38fd8f9025e6344499f9db235be98e36f +F src/json.c 4e389cbf100c53c19fff7e6fca50efb8b1ed989350eb1ae25e2950416cc9c972 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a8a9b1c89519d265869251e8b6d3c5db733f0d3a7dea6c7962811a8f1157dff -R 9be7ded4b93fb872773060703710555e -U dan -Z 20de7b64b135a6b7a6b0796593663512 +P 026618b9e321576f616a32e41329066ba629814170c6cfeef35430343f5003f3 +R 4f345097ec9904c5b67a6755cec93b7a +U drh +Z 755e762b4bd7fb34fd05935917357de5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f5d9fa1a12..33cc0bd2ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -026618b9e321576f616a32e41329066ba629814170c6cfeef35430343f5003f3 \ No newline at end of file +056de8d551dcbdf1d162e2db15ed418fa9c786f900cd3972ef8a1dea3f4f3aa1 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 42d6155fb6..5f5440ec31 100644 --- a/src/json.c +++ b/src/json.c @@ -3351,14 +3351,15 @@ static void jsonDebugPrintBlob( JsonParse *pParse, /* JSON content */ u32 iStart, /* Start rendering here */ u32 iEnd, /* Do not render this byte or any byte after this one */ - int nIndent /* Indent by this many spaces */ + int nIndent, /* Indent by this many spaces */ + sqlite3_str *pOut /* Generate output into this sqlite3_str object */ ){ while( iStartaBlob[iStart] & 0x0f; u32 savedNBlob = pParse->nBlob; - printf("%5d:%*s", iStart, nIndent, ""); + sqlite3_str_appendf(pOut, "%5d:%*s", iStart, nIndent, ""); if( pParse->nBlobAlloc>pParse->nBlob ){ pParse->nBlob = pParse->nBlobAlloc; } @@ -3367,9 +3368,11 @@ static void jsonDebugPrintBlob( if( sz>0 && xaBlob[iStart+i]); + for(i=0; iaBlob[iStart+i]); + } if( n==0 ){ - printf(" ERROR invalid node size\n"); + sqlite3_str_appendf(pOut, " ERROR invalid node size\n"); iStart = n==0 ? iStart+1 : iEnd; continue; } @@ -3384,55 +3387,57 @@ static void jsonDebugPrintBlob( } } } - printf(" <-- "); + sqlite3_str_appendall(pOut," <-- "); switch( x ){ - case JSONB_NULL: printf("null"); break; - case JSONB_TRUE: printf("true"); break; - case JSONB_FALSE: printf("false"); break; - case JSONB_INT: printf("int"); break; - case JSONB_INT5: printf("int5"); break; - case JSONB_FLOAT: printf("float"); break; - case JSONB_FLOAT5: printf("float5"); break; - case JSONB_TEXT: printf("text"); break; - case JSONB_TEXTJ: printf("textj"); break; - case JSONB_TEXT5: printf("text5"); break; - case JSONB_TEXTRAW: printf("textraw"); break; + case JSONB_NULL: sqlite3_str_appendall(pOut,"null"); break; + case JSONB_TRUE: sqlite3_str_appendall(pOut,"true"); break; + case JSONB_FALSE: sqlite3_str_appendall(pOut,"false"); break; + case JSONB_INT: sqlite3_str_appendall(pOut,"int"); break; + case JSONB_INT5: sqlite3_str_appendall(pOut,"int5"); break; + case JSONB_FLOAT: sqlite3_str_appendall(pOut,"float"); break; + case JSONB_FLOAT5: sqlite3_str_appendall(pOut,"float5"); break; + case JSONB_TEXT: sqlite3_str_appendall(pOut,"text"); break; + case JSONB_TEXTJ: sqlite3_str_appendall(pOut,"textj"); break; + case JSONB_TEXT5: sqlite3_str_appendall(pOut,"text5"); break; + case JSONB_TEXTRAW: sqlite3_str_appendall(pOut,"textraw"); break; case JSONB_ARRAY: { - printf("array, %u bytes\n", sz); - jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2); + sqlite3_str_appendf(pOut,"array, %u bytes\n", sz); + jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2, pOut); showContent = 0; break; } case JSONB_OBJECT: { - printf("object, %u bytes\n", sz); - jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2); + sqlite3_str_appendf(pOut, "object, %u bytes\n", sz); + jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2, pOut); showContent = 0; break; } default: { - printf("ERROR: unknown node type\n"); + sqlite3_str_appendall(pOut, "ERROR: unknown node type\n"); showContent = 0; break; } } if( showContent ){ if( sz==0 && x<=JSONB_FALSE ){ - printf("\n"); + sqlite3_str_append(pOut, "\n", 1); }else{ u32 i; - printf(": \""); + sqlite3_str_appendall(pOut, ": \""); for(i=iStart+n; iaBlob[i]; if( c<0x20 || c>=0x7f ) c = '.'; - putchar(c); + sqlite3_str_append(pOut, (char*)&c, 1); } - printf("\"\n"); + sqlite3_str_append(pOut, "\"\n", 2); } } iStart += n + sz; } } static void jsonShowParse(JsonParse *pParse){ + sqlite3_str out; + char zBuf[1000]; if( pParse==0 ){ printf("NULL pointer\n"); return; @@ -3443,7 +3448,10 @@ static void jsonShowParse(JsonParse *pParse){ if( pParse->nBlob==0 ) return; printf("content (bytes 0..%u):\n", pParse->nBlob-1); } - jsonDebugPrintBlob(pParse, 0, pParse->nBlob, 0); + sqlite3StrAccumInit(&out, 0, zBuf, sizeof(zBuf), 1000000); + jsonDebugPrintBlob(pParse, 0, pParse->nBlob, 0, &out); + printf("%s", sqlite3_str_value(&out)); + sqlite3_str_reset(&out); } #endif /* SQLITE_DEBUG */ @@ -3451,8 +3459,8 @@ static void jsonShowParse(JsonParse *pParse){ /* ** SQL function: json_parse(JSON) ** -** Parse JSON using jsonParseFuncArg(). Then print a dump of that -** parse on standard output. +** Parse JSON using jsonParseFuncArg(). Return text that is a +** human-readable dump of the binary JSONB for the input parameter. */ static void jsonParseFunc( sqlite3_context *ctx, @@ -3460,10 +3468,18 @@ static void jsonParseFunc( sqlite3_value **argv ){ JsonParse *p; /* The parse */ + sqlite3_str out; - assert( argc==1 ); + assert( argc>=1 ); + sqlite3StrAccumInit(&out, 0, 0, 0, 1000000); p = jsonParseFuncArg(ctx, argv[0], 0); - jsonShowParse(p); + if( p==0 ) return; + if( argc==1 ){ + jsonDebugPrintBlob(p, 0, p->nBlob, 0, &out); + sqlite3_result_text64(ctx, out.zText, out.nChar, sqlite3_free, SQLITE_UTF8); + }else{ + jsonShowParse(p); + } jsonParseFree(p); } #endif /* SQLITE_DEBUG */ From 42203e1b757154b0eed44fec8843c8771c1d9a0b Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Dec 2023 16:25:48 +0000 Subject: [PATCH 02/24] Fix harmless compiler warnings in FTS5. FossilOrigin-Name: 3cd5ef44e40570c357f913a9483fa1cd72e7f2827a5ed5826bff99febae213b1 --- ext/fts5/fts5_expr.c | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 3a7e1d5e82..05c1b59c14 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1887,7 +1887,7 @@ int sqlite3Fts5ExprClonePhrase( Fts5Expr **ppNew ){ int rc = SQLITE_OK; /* Return code */ - Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ + Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */ if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ @@ -1908,7 +1908,7 @@ int sqlite3Fts5ExprClonePhrase( pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){ Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; if( pColsetOrig ){ sqlite3_int64 nByte; diff --git a/manifest b/manifest index 0aaf21b3f9..adcf45e597 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\s(undocumented,\sdebug-only)\sjson_parse()\sSQL\sfunction\sso\sthat\sit\nreturns\sthe\stext\srendering\sof\sthe\sJSONB\sparse\sof\sthe\sinput,\srather\sthan\sprinting\nthe\srendering\son\sstdout. -D 2023-12-28T16:21:22.018 +C Fix\sharmless\scompiler\swarnings\sin\sFTS5. +D 2023-12-28T16:25:48.632 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -95,7 +95,7 @@ F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880 F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c e5fb9dd9e31e9e6ae9604bdb0d183ecec720964f3b974fc37c43ce73d8833d6d +F ext/fts5/fts5_expr.c e91156ebdcc08d837f4f324168f69f3c0d7fdef0e521fd561efb48ef3297b696 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c bb1965c3965f6fe5f64160bf1c0694a9684a790a783f293a76da1d38d319b258 F ext/fts5/fts5_main.c 93e5bc8676dbaaec365576d26cdafc5c21f0052f8f68b86ee4de22237c2e2143 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 026618b9e321576f616a32e41329066ba629814170c6cfeef35430343f5003f3 -R 4f345097ec9904c5b67a6755cec93b7a +P 056de8d551dcbdf1d162e2db15ed418fa9c786f900cd3972ef8a1dea3f4f3aa1 +R a054d36d3399814c54294e63ea00ca79 U drh -Z 755e762b4bd7fb34fd05935917357de5 +Z 9a8f01391d6842f16881c897399abbc0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 33cc0bd2ef..51b685b267 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -056de8d551dcbdf1d162e2db15ed418fa9c786f900cd3972ef8a1dea3f4f3aa1 \ No newline at end of file +3cd5ef44e40570c357f913a9483fa1cd72e7f2827a5ed5826bff99febae213b1 \ No newline at end of file From 4c13d3ca92d5214648cac9b5ae826deb7f5f16ff Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Dec 2023 19:18:08 +0000 Subject: [PATCH 03/24] Performance improvement by unwinding a loop in jsonAppendString(). FossilOrigin-Name: 190ab3c08431a0ba24d76392eab251f5c1792add05e4ec780998b299208eca95 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 29 +++++++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index adcf45e597..4c5f6a76b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sFTS5. -D 2023-12-28T16:25:48.632 +C Performance\simprovement\sby\sunwinding\sa\sloop\sin\sjsonAppendString(). +D 2023-12-28T19:18:08.833 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -697,7 +697,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 4e389cbf100c53c19fff7e6fca50efb8b1ed989350eb1ae25e2950416cc9c972 +F src/json.c a84de35816b8aebf6a97e08887cce486c0591d0b6147088fa888325f0c17f9ef F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 056de8d551dcbdf1d162e2db15ed418fa9c786f900cd3972ef8a1dea3f4f3aa1 -R a054d36d3399814c54294e63ea00ca79 +P 3cd5ef44e40570c357f913a9483fa1cd72e7f2827a5ed5826bff99febae213b1 +R 1e6a2dd6f87f9d0628f571e967da7d98 U drh -Z 9a8f01391d6842f16881c897399abbc0 +Z 7e6ac710b6732b62be341bd93538e4ea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 51b685b267..525d4ece4a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3cd5ef44e40570c357f913a9483fa1cd72e7f2827a5ed5826bff99febae213b1 \ No newline at end of file +190ab3c08431a0ba24d76392eab251f5c1792add05e4ec780998b299208eca95 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 5f5440ec31..6ee51d39ca 100644 --- a/src/json.c +++ b/src/json.c @@ -627,8 +627,33 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = '"'; while( 1 /*exit-by-break*/ ){ k = 0; - while( k+1=N ){ + while( k=N ){ if( k>0 ){ memcpy(&p->zBuf[p->nUsed], z, k); From 751c4e21526a8d04a45bd5780d54951779211fb0 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 28 Dec 2023 20:54:12 +0000 Subject: [PATCH 04/24] Update fts5origintext4.test to work with SQLITE_DIRECT_OVERFLOW_READ. FossilOrigin-Name: 15ed002aed12556aeb9bbe537c4ba839f0c95bac65a69d03401b37cc3fd11b92 --- ext/fts5/test/fts5origintext4.test | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/fts5/test/fts5origintext4.test b/ext/fts5/test/fts5origintext4.test index b9a889a514..c4ae350117 100644 --- a/ext/fts5/test/fts5origintext4.test +++ b/ext/fts5/test/fts5origintext4.test @@ -36,6 +36,7 @@ if {[permutation]=="memsubsys1" || [permutation]=="mmap"} { sqlite3_fts5_register_origintext db do_execsql_test 1.0 { + PRAGMA page_size = 4096; CREATE VIRTUAL TABLE ft USING fts5( x, tokenize="origintext unicode61", tokendata=1 ); diff --git a/manifest b/manifest index 4c5f6a76b6..38a6bc067f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovement\sby\sunwinding\sa\sloop\sin\sjsonAppendString(). -D 2023-12-28T19:18:08.833 +C Update\sfts5origintext4.test\sto\swork\swith\sSQLITE_DIRECT_OVERFLOW_READ. +D 2023-12-28T20:54:12.871 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -195,7 +195,7 @@ F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842 F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39 F ext/fts5/test/fts5origintext2.test f3b9436de540828d01f0672df855b09ebc0863e126d5b56234701d71dfa73634 F ext/fts5/test/fts5origintext3.test 0d25933506600452a5ab3873cbb418ed5f2de2446c3672b9997b1ea104b0e7f0 -F ext/fts5/test/fts5origintext4.test a33e8f64b9762e0e0c722ac2b301017e1d7745635724c1ca04b2c010b451fab4 +F ext/fts5/test/fts5origintext4.test 0c4e4514b68d9ddb15e5a538d9d234da85747a3fd62432265dbdba5c8708e457 F ext/fts5/test/fts5origintext5.test a037bdf7235a22033c4663837bdb12d9738245464a3ac2f60c71fc40d07ede7d F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3cd5ef44e40570c357f913a9483fa1cd72e7f2827a5ed5826bff99febae213b1 -R 1e6a2dd6f87f9d0628f571e967da7d98 -U drh -Z 7e6ac710b6732b62be341bd93538e4ea +P 190ab3c08431a0ba24d76392eab251f5c1792add05e4ec780998b299208eca95 +R 4943b60b261f8c7a6ac606918009691b +U dan +Z fb03567f575be0bb5e8370f46dac37ef # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 525d4ece4a..df66e453ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -190ab3c08431a0ba24d76392eab251f5c1792add05e4ec780998b299208eca95 \ No newline at end of file +15ed002aed12556aeb9bbe537c4ba839f0c95bac65a69d03401b37cc3fd11b92 \ No newline at end of file From 2aae3a95a97e38d82ec89d8b63ff5a9255b12225 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Dec 2023 21:02:08 +0000 Subject: [PATCH 05/24] Enable SQLITE_DIRECT_OVERFLOW_READ unless it is specifically disabled using the -DSQLITE_DIRECT_OVERFLOW_READ=0 compile-time option. FossilOrigin-Name: 630604a4e604bfb36c31602917bfa8d42c10c82966d0819932bf8f827b9158b8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 13 +++++++++++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 38a6bc067f..5d634df3b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sfts5origintext4.test\sto\swork\swith\sSQLITE_DIRECT_OVERFLOW_READ. -D 2023-12-28T20:54:12.871 +C Enable\sSQLITE_DIRECT_OVERFLOW_READ\sunless\sit\sis\sspecifically\sdisabled\susing\nthe\s-DSQLITE_DIRECT_OVERFLOW_READ=0\scompile-time\soption. +D 2023-12-28T21:02:08.508 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -742,7 +742,7 @@ F src/shell.c.in 85f8d52fa4f7773823736dd39d0a268fd739207fcae95883c9ec8ce4af59f7d F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 134457f62bb1d0ff1dd037cc23dd46b1d16efbbfc2211dc2b15c380af731d9ac +F src/sqliteInt.h 4552ac3dbe5be857817cff0855cdabbd20db0252739f463b5a193117c8880a02 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 190ab3c08431a0ba24d76392eab251f5c1792add05e4ec780998b299208eca95 -R 4943b60b261f8c7a6ac606918009691b -U dan -Z fb03567f575be0bb5e8370f46dac37ef +P 15ed002aed12556aeb9bbe537c4ba839f0c95bac65a69d03401b37cc3fd11b92 +R dd6052667512259356a7715bc34535d0 +U drh +Z 55f29f3b5d27fdb56a096a8d4f3db641 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index df66e453ac..d0393eb237 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15ed002aed12556aeb9bbe537c4ba839f0c95bac65a69d03401b37cc3fd11b92 \ No newline at end of file +630604a4e604bfb36c31602917bfa8d42c10c82966d0819932bf8f827b9158b8 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a21425a144..75ac749c0d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -328,6 +328,19 @@ # undef SQLITE_USE_SEH #endif +/* +** Enable SQLITE_DIRECT_OVERFLOW_READ, unless the build explicitly +** disables it using -DSQLITE_DIRECT_OVERFLOW_READ=0 +*/ +#if defined(SQLITE_DIRECT_OVERFLOW_READ) && SQLITE_DIRECT_OVERFLOW_READ+1==1 + /* Disable if -DSQLITE_DIRECT_OVERFLOW_READ=0 */ +# undef SQLITE_DIRECT_OVERFLOW_READ +#else + /* In all other cases, enable */ +# define SQLITE_DIRECT_OVERFLOW_READ 1 +#endif + + /* ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. ** 0 means mutexes are permanently disable and the library is never From e24cd27c39de2e6aade490042248af1bc67dfe19 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 29 Dec 2023 04:29:07 +0000 Subject: [PATCH 06/24] Minor doc touchup in the JS bits. FossilOrigin-Name: 8d2120c35425081e2158d6a8a6b083c4adf8d694046b2d98f5fd235520920432 --- ext/wasm/jaccwabyt/jaccwabyt.md | 5 ++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index dd80ed1c68..431741edca 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -206,9 +206,8 @@ const MyBinder = StructBinderFactory({ It also offers a number of other settings, but all are optional except for the ones shown above. Those three config options abstract away details which are specific to a given WASM environment. They provide -the WASM "heap" memory (a byte array), the memory allocator, and the -deallocator. In a conventional Emscripten setup, that config might -simply look like: +the WASM "heap" memory, the memory allocator, and the deallocator. In +a conventional Emscripten setup, that config might simply look like: > ```javascript diff --git a/manifest b/manifest index 5d634df3b0..adb892f498 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sSQLITE_DIRECT_OVERFLOW_READ\sunless\sit\sis\sspecifically\sdisabled\susing\nthe\s-DSQLITE_DIRECT_OVERFLOW_READ=0\scompile-time\soption. -D 2023-12-28T21:02:08.508 +C Minor\sdoc\stouchup\sin\sthe\sJS\sbits. +D 2023-12-29T04:29:07.096 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81 F ext/wasm/index-dist.html e91d76e4581185238fd3d42ed86ec600f7023ed3e3a944c5c356f25304bf1263 F ext/wasm/index.html b31ce41c0da476d5ffcef23069b9d3415b419d65af5779096ebcfbcbade453a9 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54 -F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb +F ext/wasm/jaccwabyt/jaccwabyt.md 59a20df389abcc3606eb4eaea7fb7ba14504beb3e345dbea9b99a0618ba3bec8 F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 15ed002aed12556aeb9bbe537c4ba839f0c95bac65a69d03401b37cc3fd11b92 -R dd6052667512259356a7715bc34535d0 -U drh -Z 55f29f3b5d27fdb56a096a8d4f3db641 +P 630604a4e604bfb36c31602917bfa8d42c10c82966d0819932bf8f827b9158b8 +R 0182e48a4eb943a57927e95defc2f847 +U stephan +Z 19ca2e7863b05c35a59e90f21d5378b3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d0393eb237..c9aa0e6dea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -630604a4e604bfb36c31602917bfa8d42c10c82966d0819932bf8f827b9158b8 \ No newline at end of file +8d2120c35425081e2158d6a8a6b083c4adf8d694046b2d98f5fd235520920432 \ No newline at end of file From 7a482b3bcda5c36e0c50f05921ddfbb3256fc64a Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 31 Dec 2023 04:01:36 +0000 Subject: [PATCH 07/24] Use SQLITE_ENABLE_STAT4 in both the WASM and JNI builds. FossilOrigin-Name: 99d11e6d0ae687ff6bac5119027f7b04d5e7185214e79cf8c56289cfa809b0f9 --- ext/jni/src/c/sqlite3-jni.c | 3 +++ ext/wasm/api/sqlite3-wasm.c | 3 +++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index 5deff19ef1..64a2bef198 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -76,6 +76,9 @@ //#ifndef SQLITE_ENABLE_SESSION //# define SQLITE_ENABLE_SESSION 1 //#endif +#ifndef SQLITE_ENABLE_STAT4 +# define SQLITE_ENABLE_STAT4 1 +#endif #ifndef SQLITE_ENABLE_STMTVTAB # define SQLITE_ENABLE_STMTVTAB 1 #endif diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 3e22c799c4..23386c7c70 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -122,6 +122,9 @@ #ifndef SQLITE_ENABLE_SESSION # define SQLITE_ENABLE_SESSION 1 #endif +#ifndef SQLITE_ENABLE_STAT4 +# define SQLITE_ENABLE_STAT4 1 +#endif #ifndef SQLITE_ENABLE_STMTVTAB # define SQLITE_ENABLE_STMTVTAB 1 #endif diff --git a/manifest b/manifest index adb892f498..cdba3da231 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sdoc\stouchup\sin\sthe\sJS\sbits. -D 2023-12-29T04:29:07.096 +C Use\sSQLITE_ENABLE_STAT4\sin\sboth\sthe\sWASM\sand\sJNI\sbuilds. +D 2023-12-31T04:01:36.325 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -251,7 +251,7 @@ F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a3 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa -F ext/jni/src/c/sqlite3-jni.c c1292e690a20c7787a63e8d8ac6e2dfed49c97282ed056a7cfda5da461f0b7d8 +F ext/jni/src/c/sqlite3-jni.c a477b2d0216550cdd6f248c92e4fd666ec2177a350a67417e0b5b9840f52e1c5 F ext/jni/src/c/sqlite3-jni.h 913ab8e8fee432ae40f0e387c8231118d17053714703f5ded18202912a8a3fbf F ext/jni/src/org/sqlite/jni/annotation/Experimental.java 8603498634e41d0f7c70f661f64e05df64376562ea8f126829fd1e0cdd47e82b F ext/jni/src/org/sqlite/jni/annotation/NotNull.java 38e7e58a69b26dc100e458b31dfa3b2a7d67bc36d051325526ef1987d5bc8a24 @@ -602,7 +602,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 595953994aa3ae2287c889c4da39ab3d6f17b6461ecf4bec334b7a3faafddb02 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91ccfafc38b400fe36e90bdda88e -F ext/wasm/api/sqlite3-wasm.c f280d4ea917d213ae95668dfcd173a2c2ef21a0a4bf9aeb9fcd0edaf1b21ba4b +F ext/wasm/api/sqlite3-wasm.c dbaefda7981050bf9eb413f5b24df648ead09d9a3e796f9ba84d87cb49b5ab80 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js a94e3f0ca25a777bb73779368f97be0e103e02d067ad3ee3e9c4cc5bcefbd01c F ext/wasm/api/sqlite3-worker1.c-pp.js a541112aa51e16705f13a99bb943c64efe178aa28c86704a955f8fd9afe4ba37 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 630604a4e604bfb36c31602917bfa8d42c10c82966d0819932bf8f827b9158b8 -R 0182e48a4eb943a57927e95defc2f847 +P 8d2120c35425081e2158d6a8a6b083c4adf8d694046b2d98f5fd235520920432 +R 0fc8decc05e314121b9845282e3b044a U stephan -Z 19ca2e7863b05c35a59e90f21d5378b3 +Z c3f0712f8c19c3c3ba1e6e96354954de # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c9aa0e6dea..3cf768fb4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d2120c35425081e2158d6a8a6b083c4adf8d694046b2d98f5fd235520920432 \ No newline at end of file +99d11e6d0ae687ff6bac5119027f7b04d5e7185214e79cf8c56289cfa809b0f9 \ No newline at end of file From 805cd395a988d78843173a9e8fe1eccfc7ec3f71 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 1 Jan 2024 05:58:47 +0000 Subject: [PATCH 08/24] WASM: various build cleanups and add initial infrastructure for a build which elides the oo1 API and its dependents (worker1 and promiser). Sidebar: an attempt was made to move generation of the build rules to an external script, but the mixed-mode make/script was even less legible than the $(eval) indirection going on in the makefile. FossilOrigin-Name: 563d313163c02b398ae85b7c2ed231019a14e006726f09a7c1f294a58bf4363f --- ext/wasm/GNUmakefile | 61 +++++++++++++------ ext/wasm/api/sqlite3-api-oo1.js | 5 +- ext/wasm/api/sqlite3-api-worker1.js | 4 ++ ext/wasm/api/sqlite3-worker1-promiser.c-pp.js | 4 ++ ext/wasm/api/sqlite3-worker1.c-pp.js | 4 ++ manifest | 20 +++--- manifest.uuid | 2 +- 7 files changed, 69 insertions(+), 31 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 0510af74ab..94eff4cc32 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -54,6 +54,9 @@ release: oz # JS_BUILD_MODES exists solely to reduce repetition in documentation # below. JS_BUILD_MODES := vanilla esm bunder-friendly node +# JS_BUILD_NAMES exists solely to reduce repetition in documentation +# below. +JS_BUILD_NAMES := sqlite3 sqlite3-wasmfs # Emscripten SDK home dir and related binaries... EMSDK_HOME ?= $(word 1,$(wildcard $(HOME)/emsdk $(HOME)/src/emsdk)) emcc.bin ?= $(word 1,$(wildcard $(EMSDK_HOME)/upstream/emscripten/emcc) $(shell which emcc)) @@ -287,6 +290,9 @@ DISTCLEAN_FILES += $(bin.stripccomments) # c-pp.c was written specifically for the sqlite project's JavaScript # builds but is maintained as a standalone project: # https://fossil.wanderinghorse.net/r/c-pp +# +# Note that the SQLITE_... build flags used here have NO EFFECT on the +# JS/WASM build. They are solely for use with $(bin.c-pp) itself. bin.c-pp := ./c-pp $(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ @@ -518,13 +524,14 @@ emcc.jsflags += -sSTACK_SIZE=512KB # extern-post-js.js. However... using a temporary symbol name here # and then adding sqlite3InitModule() ourselves results in 2 global # symbols: we cannot "delete" the Emscripten-defined -# $(sqlite3.js.init-func) because it's declared with "var". +# $(sqlite3.js.init-func) from vanilla builds (as opposed to ESM +# builds) because it's declared with "var". sqlite3.js.init-func := sqlite3InitModule emcc.jsflags += -sEXPORT_NAME=$(sqlite3.js.init-func) emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. #emcc.jsflags += -sSTRICT # fails due to missing __syscall_...() #emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS -#emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API +#emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. fiddle needs the FS API #emcc.jsflags += -sABORTING_MALLOC # only for experimentation emcc.jsflags += -sALLOW_TABLE_GROWTH # ^^^^ -sALLOW_TABLE_GROWTH is required for installing new SQL UDFs @@ -568,15 +575,20 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED # -g3 debugging info, _huge_. ######################################################################## +# $(sqlite3-api-build-version.js) injects the build version info into +# the bundle. $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) @echo "Making $@..." @{ \ - echo 'globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){'; \ - echo -n ' sqlite3.version = '; \ - $(bin.version-info) --json; \ - echo ';'; \ - echo '});'; \ + echo 'globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){'; \ + echo -n ' sqlite3.version = '; \ + $(bin.version-info) --json; \ + echo ';'; \ + echo '});'; \ } > $@ + +# $(sqlite3-license-version.js) contains the license header and build +# version info. $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) \ $(MAKEFILE) @echo "Making $@..."; { \ @@ -594,7 +606,11 @@ $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) ######################################################################## # --post-js and --pre-js are emcc flags we use to append/prepend JS to # the generated emscripten module file. These rules set up the core -# pre/post files for use by the various builds. +# pre/post files for use by the various builds. --pre-js is used to +# inject code which needs to run as part of the pre-WASM-load phase. +# --post-js injects code which runs after the WASM module is loaded +# and includes the entirety of the library plus some +# Emscripten-specific post-bootstrapping code. pre-js.js.in := $(dir.api)/pre-js.c-pp.js post-js.js.in := $(dir.tmp)/post-js.c-pp.js post-jses.js := \ @@ -612,18 +628,26 @@ $(post-js.js.in): $(post-jses.js) $(MAKEFILE) ######################################################################## # call-make-pre-post is a $(call)able which creates rules for -# pre-js-$(1)-$(2).js. $1 = the base name of the JS file on whose -# behalf this pre-js is for (one of: sqlite3, sqlite3-wasmfs). $2 is +# pre-js.$(1)-$(2).js. $1 = the base name of the JS file on whose +# behalf this pre-js is for (one of: $(JS_BUILD_NAMES)). $2 is # the build mode: one of $(JS_BUILD_MODES). This sets up # --[extern-][pre/post]-js flags in $(pre-post-$(1)-$(2).flags) and # dependencies in $(pre-post-$(1)-$(2).deps). The resulting files get # filtered using $(C-PP.FILTER). Any flags necessary for such # filtering need to be set in $(c-pp.D.$(1)-$(2)) before $(call)ing # this. +# +# Maintenance note: a shell script was written to generate these rules +# with the hope that it would make them more legible and maintainable, +# but embedding makefile code in another language makes it even less +# legible than having the level of $(eval) indirection which we have +# here. define call-make-pre-post pre-post-$(1)-$(2).flags ?= -pre-js.js.$(1)-$(2) := $$(dir.tmp)/pre-js.$(1)-$(2).intermediary.js -$$(eval $$(call C-PP.FILTER,$$(pre-js.js.in),$$(pre-js.js.$(1)-$(2)),$$(c-pp.D.$(1)-$(2)))) +pre-js.js.$(1)-$(2).intermediary := $$(dir.tmp)/pre-js.$(1)-$(2).intermediary.js +pre-js.js.$(1)-$(2) := $$(dir.tmp)/pre-js.$(1)-$(2).js +#$$(error $$(pre-js.js.$(1)-$(2).intermediary) $$(pre-js.js.$(1)-$(2))) +$$(eval $$(call C-PP.FILTER,$$(pre-js.js.in),$$(pre-js.js.$(1)-$(2).intermediary),$$(c-pp.D.$(1)-$(2)))) post-js.js.$(1)-$(2) := $$(dir.tmp)/post-js.$(1)-$(2).js $$(eval $$(call C-PP.FILTER,$$(post-js.js.in),$$(post-js.js.$(1)-$(2)),$$(c-pp.D.$(1)-$(2)))) extern-post-js.js.$(1)-$(2) := $$(dir.tmp)/extern-post-js.$(1)-$(2).js @@ -634,8 +658,8 @@ pre-post-common.flags.$(1)-$(2) := \ --extern-post-js=$$(extern-post-js.js.$(1)-$(2)) pre-post-jses.$(1)-$(2).deps := $$(pre-post-jses.deps.common) \ $$(post-js.js.$(1)-$(2)) $$(extern-post-js.js.$(1)-$(2)) -$$(dir.tmp)/pre-js-$(1)-$(2).js: $$(pre-js.js.$(1)-$(2)) $$(MAKEFILE) - cp $$(pre-js.js.$(1)-$(2)) $$@ +$$(pre-js.js.$(1)-$(2)): $$(pre-js.js.$(1)-$(2).intermediary) $$(MAKEFILE) + cp $$(pre-js.js.$(1)-$(2).intermediary) $$@ @if [ sqlite3-wasmfs = $(1) ]; then \ echo "delete Module[xNameOfInstantiateWasm] /*for WASMFS build*/;"; \ elif [ sqlite3 != $(1) ]; then \ @@ -643,10 +667,10 @@ $$(dir.tmp)/pre-js-$(1)-$(2).js: $$(pre-js.js.$(1)-$(2)) $$(MAKEFILE) fi >> $$@ pre-post-$(1)-$(2).deps := \ $$(pre-post-jses.$(1)-$(2).deps) \ - $$(dir.tmp)/pre-js-$(1)-$(2).js + $$(dir.tmp)/pre-js.$(1)-$(2).js pre-post-$(1)-$(2).flags += \ $$(pre-post-common.flags.$(1)-$(2)) \ - --pre-js=$$(dir.tmp)/pre-js-$(1)-$(2).js + --pre-js=$$(dir.tmp)/pre-js.$(1)-$(2).js endef # /post-js and pre-js ######################################################################## @@ -715,7 +739,7 @@ pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) # SETUP_LIB_BUILD_MODE is a $(call)'able which sets up numerous pieces # for one of the build modes. # -# $1 = one of: sqlite3, sqlite3-wasmfs +# $1 = one of: $(JS_BUILD_NAMES) # $2 = build mode name: one of $(JS_BUILD_MODES) # $3 = 1 for ESM build mode, else 0 # $4 = resulting sqlite-api JS/MJS file @@ -781,8 +805,7 @@ sqlite3-node.mjs := $(dir.dout)/sqlite3-node.mjs $(eval $(call SETUP_LIB_BUILD_MODE,sqlite3,vanilla,0,\ $(sqlite3-api.js), $(sqlite3.js))) $(eval $(call SETUP_LIB_BUILD_MODE,sqlite3,esm,1,\ - $(sqlite3-api.mjs), $(sqlite3.mjs), \ - -Dtarget=es6-module, -sEXPORT_ES6 -sUSE_ES6_IMPORT_META)) + $(sqlite3-api.mjs), $(sqlite3.mjs), -Dtarget=es6-module)) $(eval $(call SETUP_LIB_BUILD_MODE,sqlite3,bundler-friendly,1,\ $(sqlite3-api-bundler-friendly.mjs),$(sqlite3-bundler-friendly.mjs),\ $(c-pp.D.sqlite3-esm) -Dtarget=es6-bundler-friendly)) diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 4677b89762..160d59db5a 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -1,3 +1,4 @@ +//#ifnot omit-oo1 /* 2022-07-22 @@ -1940,4 +1941,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*main-window-only bits*/ }); - +//#else +/* Built with the omit-oo1 flag. */ +//#endif ifnot omit-oo1 diff --git a/ext/wasm/api/sqlite3-api-worker1.js b/ext/wasm/api/sqlite3-api-worker1.js index 7c65dd1d3e..3099c19ec4 100644 --- a/ext/wasm/api/sqlite3-api-worker1.js +++ b/ext/wasm/api/sqlite3-api-worker1.js @@ -1,3 +1,4 @@ +//#ifnot omit-oo1 /** 2022-07-22 @@ -689,3 +690,6 @@ sqlite3.initWorker1API = function(){ globalThis.postMessage({type:'sqlite3-api',result:'worker1-ready'}); }.bind({sqlite3}); }); +//#else +/* Built with the omit-oo1 flag. */ +//#endif ifnot omit-oo1 diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index d4445cdcdf..cd78ed4bcf 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -1,3 +1,4 @@ +//#ifnot omit-oo1 /* 2022-08-24 @@ -276,3 +277,6 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = { , onerror: (...args)=>console.error('worker1 promiser error',...args) }; +//#else +/* Built with the omit-oo1 flag. */ +//#endif ifnot omit-oo1 diff --git a/ext/wasm/api/sqlite3-worker1.c-pp.js b/ext/wasm/api/sqlite3-worker1.c-pp.js index 220722ffe1..74de9ec7ef 100644 --- a/ext/wasm/api/sqlite3-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1.c-pp.js @@ -1,3 +1,4 @@ +//#ifnot omit-oo1 /* 2022-05-23 @@ -48,3 +49,6 @@ import {default as sqlite3InitModule} from './sqlite3-bundler-friendly.mjs'; } //#endif sqlite3InitModule().then(sqlite3 => sqlite3.initWorker1API()); +//#else +/* Built with the omit-oo1 flag. */ +//#endif ifnot omit-oo1 diff --git a/manifest b/manifest index cdba3da231..579bc6df6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sSQLITE_ENABLE_STAT4\sin\sboth\sthe\sWASM\sand\sJNI\sbuilds. -D 2023-12-31T04:01:36.325 +C WASM:\svarious\sbuild\scleanups\sand\sadd\sinitial\sinfrastructure\sfor\sa\sbuild\swhich\selides\sthe\soo1\sAPI\sand\sits\sdependents\s(worker1\sand\spromiser).\sSidebar:\san\sattempt\swas\smade\sto\smove\sgeneration\sof\sthe\sbuild\srules\sto\san\sexternal\sscript,\sbut\sthe\smixed-mode\smake/script\swas\seven\sless\slegible\sthan\sthe\s$(eval)\sindirection\sgoing\son\sin\sthe\smakefile. +D 2024-01-01T05:58:47.485 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 62403519b233dbe23e1cac30969714c4043a96c5bc2614e551a07a81c543c493 +F ext/wasm/GNUmakefile a8a171af359b9acee604bb01229f59052b8b38c43f6020a6d009e42c76f910af F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -594,17 +594,17 @@ F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e F ext/wasm/api/sqlite3-api-glue.js 119b91c8a7ce6648679eb66fcdd1ed07ef7fd892eb501d658fbfefcc962012d9 -F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 +F ext/wasm/api/sqlite3-api-oo1.js 7f3bcf0549ac44cde4b9da0b642d771916738d3f6781fb8a1757c50a91e506c0 F ext/wasm/api/sqlite3-api-prologue.js 9aeba7b45cf41b3a26d34d7fb2525633cd1adfc544888c1ea8dbb077496f4ce9 -F ext/wasm/api/sqlite3-api-worker1.js 88770ac01fc756f89a3e060eec17111d6c1688e89ebfd34cb9d9e54d25affbb9 +F ext/wasm/api/sqlite3-api-worker1.js fd46628ef147dd5856c88f63a9a279a40f744f1fdfddd55251ad8fbc3d8200ae F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 595953994aa3ae2287c889c4da39ab3d6f17b6461ecf4bec334b7a3faafddb02 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91ccfafc38b400fe36e90bdda88e F ext/wasm/api/sqlite3-wasm.c dbaefda7981050bf9eb413f5b24df648ead09d9a3e796f9ba84d87cb49b5ab80 -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js a94e3f0ca25a777bb73779368f97be0e103e02d067ad3ee3e9c4cc5bcefbd01c -F ext/wasm/api/sqlite3-worker1.c-pp.js a541112aa51e16705f13a99bb943c64efe178aa28c86704a955f8fd9afe4ba37 +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js f234191fe6bf41a5a1e59c9f43ed816e74a522b3d60d3f556f66c3085c448503 +F ext/wasm/api/sqlite3-worker1.c-pp.js 5e8706c2c4af2a57fbcdc02f4e7ef79869971bc21bb8ede777687786ce1c92d5 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d2120c35425081e2158d6a8a6b083c4adf8d694046b2d98f5fd235520920432 -R 0fc8decc05e314121b9845282e3b044a +P 99d11e6d0ae687ff6bac5119027f7b04d5e7185214e79cf8c56289cfa809b0f9 +R f12f6c5bbcd045460cf7bd0ad6d9cb62 U stephan -Z c3f0712f8c19c3c3ba1e6e96354954de +Z 65fba1a0552f62ac5813e31c03da0bea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3cf768fb4e..99bdacd567 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -99d11e6d0ae687ff6bac5119027f7b04d5e7185214e79cf8c56289cfa809b0f9 \ No newline at end of file +563d313163c02b398ae85b7c2ed231019a14e006726f09a7c1f294a58bf4363f \ No newline at end of file From 307f326ba3d448c55abd04e241d46af903667653 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 1 Jan 2024 06:58:32 +0000 Subject: [PATCH 09/24] JNI: move the ByteBuffer-using APIs from public to package visibility for the time being because they have UB-inducing possibilities which need to be worked out. Update test code to account for a change in custom FTS5 columntext() impls. FossilOrigin-Name: dc501275fcfab3ad9b6ebbadf7588b225a9dd07a0abac5be83d96f15bfba99e9 --- ext/jni/src/org/sqlite/jni/capi/CApi.java | 34 +++++++++---------- .../src/org/sqlite/jni/fts5/TesterFts5.java | 18 +++++++--- manifest | 14 ++++---- manifest.uuid | 2 +- 4 files changed, 39 insertions(+), 29 deletions(-) diff --git a/ext/jni/src/org/sqlite/jni/capi/CApi.java b/ext/jni/src/org/sqlite/jni/capi/CApi.java index 89d62849c0..b5d08306e2 100644 --- a/ext/jni/src/org/sqlite/jni/capi/CApi.java +++ b/ext/jni/src/org/sqlite/jni/capi/CApi.java @@ -256,7 +256,7 @@ public final class CApi { sqlite3_bind_nio_buffer(). */ @Experimental - public static int sqlite3_bind_blob( + /*public*/ static int sqlite3_bind_blob( @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data, int begin, int n ){ @@ -269,7 +269,7 @@ public final class CApi { final two arguments. */ @Experimental - public static int sqlite3_bind_blob( + /*public*/ static int sqlite3_bind_blob( @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data ){ return sqlite3_bind_nio_buffer(stmt, ndx, data, 0, -1); @@ -346,7 +346,7 @@ public final class CApi { @see https://docs.oracle.com/javase/8/docs/api/java/nio/Buffer.html */ @Experimental - public static native int sqlite3_bind_nio_buffer( + /*public*/ static native int sqlite3_bind_nio_buffer( @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data, int beginPos, int howMany ); @@ -356,7 +356,7 @@ public final class CApi { contents, up to its limit() (as opposed to its capacity()). */ @Experimental - public static int sqlite3_bind_nio_buffer( + /*public*/ static int sqlite3_bind_nio_buffer( @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data ){ return sqlite3_bind_nio_buffer(stmt, ndx, data, 0, -1); @@ -600,7 +600,7 @@ public final class CApi { sqlite3_blob_read() (0 on success). */ @Experimental - public static int sqlite3_blob_read_nio_buffer( + /*public*/ static int sqlite3_blob_read_nio_buffer( @NotNull sqlite3_blob src, int srcOffset, @NotNull java.nio.ByteBuffer tgt, int tgtOffset, int howMany ){ @@ -624,7 +624,7 @@ public final class CApi { for any reason. */ @Experimental - public static java.nio.ByteBuffer sqlite3_blob_read_nio_buffer( + /*public*/ static java.nio.ByteBuffer sqlite3_blob_read_nio_buffer( @NotNull sqlite3_blob src, int srcOffset, int howMany ){ if( !JNI_SUPPORTS_NIO || src==null ) return null; @@ -645,7 +645,7 @@ public final class CApi { Overload alias for sqlite3_blob_read_nio_buffer(). */ @Experimental - public static int sqlite3_blob_read( + /*public*/ static int sqlite3_blob_read( @NotNull sqlite3_blob src, int srcOffset, @NotNull java.nio.ByteBuffer tgt, int tgtOffset, int howMany @@ -668,7 +668,7 @@ public final class CApi { the result of the underlying call to sqlite3_blob_read(). */ @Experimental - public static int sqlite3_blob_read( + /*public*/ static int sqlite3_blob_read( @NotNull sqlite3_blob src, @NotNull java.nio.ByteBuffer tgt ){ @@ -727,7 +727,7 @@ public final class CApi { returns the result of the underlying call to sqlite3_blob_read(). */ @Experimental - public static int sqlite3_blob_write_nio_buffer( + /*public*/ static int sqlite3_blob_write_nio_buffer( @NotNull sqlite3_blob tgt, int tgtOffset, @NotNull java.nio.ByteBuffer src, int srcOffset, int howMany @@ -756,7 +756,7 @@ public final class CApi { of b. */ @Experimental - public static int sqlite3_blob_write( + /*public*/ static int sqlite3_blob_write( @NotNull sqlite3_blob tgt, int tgtOffset, @NotNull java.nio.ByteBuffer src ){ @@ -770,7 +770,7 @@ public final class CApi { of tgt. */ @Experimental - public static int sqlite3_blob_write( + /*public*/ static int sqlite3_blob_write( @NotNull sqlite3_blob tgt, @NotNull java.nio.ByteBuffer src ){ @@ -926,7 +926,7 @@ public final class CApi { would return null for the same inputs. */ @Experimental - public static native java.nio.ByteBuffer sqlite3_column_nio_buffer( + /*public*/ static native java.nio.ByteBuffer sqlite3_column_nio_buffer( @NotNull sqlite3_stmt stmt, int ndx ); @@ -1848,7 +1848,7 @@ public final class CApi { then this function behaves like sqlite3_result_error_toobig(). */ @Experimental - public static native void sqlite3_result_nio_buffer( + /*public*/ static native void sqlite3_result_nio_buffer( @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob, int begin, int n ); @@ -1858,7 +1858,7 @@ public final class CApi { as the result blob content. */ @Experimental - public static void sqlite3_result_nio_buffer( + /*public*/ static void sqlite3_result_nio_buffer( @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob ){ sqlite3_result_nio_buffer(cx, blob, 0, -1); @@ -1963,7 +1963,7 @@ public final class CApi { sqlite3_result_nio_buffer(). */ @Experimental - public static void sqlite3_result_blob( + /*public*/ static void sqlite3_result_blob( @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob, int begin, int n ){ @@ -1975,7 +1975,7 @@ public final class CApi { sqlite3_result_nio_buffer(). */ @Experimental - public static void sqlite3_result_blob( + /*public*/ static void sqlite3_result_blob( @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob ){ sqlite3_result_nio_buffer(cx, blob); @@ -2387,7 +2387,7 @@ public final class CApi { would return null for the same input. */ @Experimental - public static native java.nio.ByteBuffer sqlite3_value_nio_buffer( + /*public*/ static native java.nio.ByteBuffer sqlite3_value_nio_buffer( @NotNull sqlite3_value v ); diff --git a/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java b/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java index c4264c5417..095e649caf 100644 --- a/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java +++ b/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java @@ -625,10 +625,20 @@ public class TesterFts5 { "SELECT fts5_columntext(ft, 1) FROM ft('x') ORDER BY rowid", "[x, x, x y z, x z, x y z, x]" ); - do_execsql_test(db, - "SELECT fts5_columntext(ft, 2) FROM ft('x') ORDER BY rowid", - "[null, null, null, null, null, null]" - ); + boolean threw = false; + try{ + /* columntext() used to return NULLs when given an out-of bounds column + but now results in a range error. */ + do_execsql_test(db, + "SELECT fts5_columntext(ft, 2) FROM ft('x') ORDER BY rowid", + "[null, null, null, null, null, null]" + ); + }catch(Exception e){ + threw = true; + affirm( e.getMessage().matches(".*column index out of range") ); + } + affirm( threw ); + threw = false; /* Test fts5_columntotalsize() */ do_execsql_test(db, diff --git a/manifest b/manifest index 579bc6df6d..3a70cf068a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C WASM:\svarious\sbuild\scleanups\sand\sadd\sinitial\sinfrastructure\sfor\sa\sbuild\swhich\selides\sthe\soo1\sAPI\sand\sits\sdependents\s(worker1\sand\spromiser).\sSidebar:\san\sattempt\swas\smade\sto\smove\sgeneration\sof\sthe\sbuild\srules\sto\san\sexternal\sscript,\sbut\sthe\smixed-mode\smake/script\swas\seven\sless\slegible\sthan\sthe\s$(eval)\sindirection\sgoing\son\sin\sthe\smakefile. -D 2024-01-01T05:58:47.485 +C JNI:\smove\sthe\sByteBuffer-using\sAPIs\sfrom\spublic\sto\spackage\svisibility\sfor\sthe\stime\sbeing\sbecause\sthey\shave\sUB-inducing\spossibilities\swhich\sneed\sto\sbe\sworked\sout.\sUpdate\stest\scode\sto\saccount\sfor\sa\schange\sin\scustom\sFTS5\scolumntext()\simpls. +D 2024-01-01T06:58:32.507 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -262,7 +262,7 @@ F ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java 0b72cdff61533b564d65b63 F ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java c045a5b47e02bb5f1af91973814a905f12048c428a3504fbc5266d1c1be3de5a F ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java 74cc4998a73d6563542ecb90804a3c4f4e828cb4bd69e61226d1a51f4646e759 F ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java 7b8e19810c42b0ad21a04b5d8c804b32ee5905d137148703f16a75b612c380ca -F ext/jni/src/org/sqlite/jni/capi/CApi.java d428a1fd3b827f01c55d10d21ff35e33e7dac9e8a1d92a8b5c7d7255e67407d8 +F ext/jni/src/org/sqlite/jni/capi/CApi.java 27bbd944ea8c147afd25b93f17dc397f3627611ebe2878944a32ffeffc98e99e F ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java 57e2d275dcebe690b1fc1f3d34eb96879b2d7039bce30b563aee547bf45d8a8b F ext/jni/src/org/sqlite/jni/capi/CollationCallback.java e29bcfc540fdd343e2f5cca4d27235113f2886acb13380686756d5cabdfd065a F ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java 5bfa226a8e7a92e804fd52d6e42b4c7b875fa7a94f8e2c330af8cc244a8920ab @@ -298,7 +298,7 @@ F ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java 338637e6e5a2cc385d962b220f3c1 F ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java 7da0fbb5728f7c056a43e6407f13dd0c7c9c445221267786a109b987f5fc8a9d F ext/jni/src/org/sqlite/jni/fts5/Fts5PhraseIter.java 28045042d593a1f1b9b80d54ec77cbf1d8a1bc95e442eceefa9a3a6f56600b0e F ext/jni/src/org/sqlite/jni/fts5/Fts5Tokenizer.java 3c8f677ffb85b8782f865d6fcbc16200b3375d0e3c29ed541a494fde3011bf49 -F ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java eaee4d641229a098eb704b96a45c9a23c6514dc39009d3611e265bab33834deb +F ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java 51e16bf9050af7cb246d17d6a19c001cfc916bf20f425c96625aaccaf74688e8 F ext/jni/src/org/sqlite/jni/fts5/XTokenizeCallback.java 1efd1220ea328a32f2d2a1b16c735864159e929480f71daad4de9d5944839167 F ext/jni/src/org/sqlite/jni/fts5/fts5_api.java a8e88c3783d21cec51b0748568a96653fead88f8f4953376178d9c7385b197ea F ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java 9e2b954d210d572552b28aca523b272fae14bd41e318921b22f65b728d5bf978 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 99d11e6d0ae687ff6bac5119027f7b04d5e7185214e79cf8c56289cfa809b0f9 -R f12f6c5bbcd045460cf7bd0ad6d9cb62 +P 563d313163c02b398ae85b7c2ed231019a14e006726f09a7c1f294a58bf4363f +R 58350c999f119bc20e49e20150798450 U stephan -Z 65fba1a0552f62ac5813e31c03da0bea +Z a9b067fc18aa3982ca71d8b9d5540ad9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 99bdacd567..5a0f386979 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -563d313163c02b398ae85b7c2ed231019a14e006726f09a7c1f294a58bf4363f \ No newline at end of file +dc501275fcfab3ad9b6ebbadf7588b225a9dd07a0abac5be83d96f15bfba99e9 \ No newline at end of file From 4ccf707c6ebd294f1ecf6e9f304bd083fbc3a71d Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 1 Jan 2024 17:48:02 +0000 Subject: [PATCH 10/24] Extra steps taken to avoid using low-quality indexes in a query plan. This branch accomplishes the same end as the nearby enhanced-stat1 branch, but with much less change and hence less risk. FossilOrigin-Name: c030e646262fee43a59b45fdc1630d972f8bf88ac3c142b6bdaf4cbb36695a4f --- manifest | 21 ++++++++++++--------- manifest.uuid | 2 +- src/analyze.c | 10 ++++++++++ src/sqliteInt.h | 1 + src/where.c | 5 ++++- 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 3a70cf068a..065f4dad41 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C JNI:\smove\sthe\sByteBuffer-using\sAPIs\sfrom\spublic\sto\spackage\svisibility\sfor\sthe\stime\sbeing\sbecause\sthey\shave\sUB-inducing\spossibilities\swhich\sneed\sto\sbe\sworked\sout.\sUpdate\stest\scode\sto\saccount\sfor\sa\schange\sin\scustom\sFTS5\scolumntext()\simpls. -D 2024-01-01T06:58:32.507 +C Extra\ssteps\staken\sto\savoid\susing\slow-quality\sindexes\sin\sa\squery\splan.\nThis\sbranch\saccomplishes\sthe\ssame\send\sas\sthe\snearby\senhanced-stat1\sbranch,\nbut\swith\smuch\sless\schange\sand\shence\sless\srisk. +D 2024-01-01T17:48:02.427 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 30c2333b8bb3af71e4eb9adeadee8aa20edb15917ed44b8422e5cd15f3dfcddc -F src/analyze.c d4cc28738c29e009640ec20ebb6936ba6fcefff0d11aa93398d9bb9a5ead6c1f +F src/analyze.c 64ac1dd3b7044c7a7981a5f6d3796bc007b7e31514c674283bf440586681501e F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 @@ -742,7 +742,7 @@ F src/shell.c.in 85f8d52fa4f7773823736dd39d0a268fd739207fcae95883c9ec8ce4af59f7d F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 4552ac3dbe5be857817cff0855cdabbd20db0252739f463b5a193117c8880a02 +F src/sqliteInt.h 73800d73e21180e6b3df8d0fe7d11758dc24367fd2b0b0075b48fc116de406bb F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -822,7 +822,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 0bfab37c7f787e320a8010e51ae97c2e51964d3b3a24fbc246b8e8fee50de4e9 +F src/where.c 217fe82a26c0fb6a3c7fd01865d821e752f9c01fb72f114af3f0b77ce234d1fb F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 @@ -2156,8 +2156,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 563d313163c02b398ae85b7c2ed231019a14e006726f09a7c1f294a58bf4363f -R 58350c999f119bc20e49e20150798450 -U stephan -Z a9b067fc18aa3982ca71d8b9d5540ad9 +P dc501275fcfab3ad9b6ebbadf7588b225a9dd07a0abac5be83d96f15bfba99e9 +R 4e0f7b3b61e72c244a187959e2e9f486 +T *branch * avoid-low-quality-indexes +T *sym-avoid-low-quality-indexes * +T -sym-trunk * +U drh +Z ac10d23b30667bc4467501c1b744109a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5a0f386979..c5640b9f7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc501275fcfab3ad9b6ebbadf7588b225a9dd07a0abac5be83d96f15bfba99e9 \ No newline at end of file +c030e646262fee43a59b45fdc1630d972f8bf88ac3c142b6bdaf4cbb36695a4f \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index a7a8b6d665..a07f1939c7 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1556,6 +1556,16 @@ static void decodeIntArray( while( z[0]!=0 && z[0]!=' ' ) z++; while( z[0]==' ' ) z++; } + + /* Set the bLowQual flag if the peak number of rows obtained + ** from a full equality match is so large that a full table scan + ** seems likely to be faster than using the index. + */ + if( aLog[0] > 66 /* Index has more than 100 rows */ + && aLog[0] <= aLog[nOut-1] /* And only a single value seen */ + ){ + pIndex->bLowQual = 1; + } } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 75ac749c0d..79a36e060e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2766,6 +2766,7 @@ struct Index { unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ + unsigned bLowQual:1; /* sqlite_stat1 says this is a low-quality index */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ diff --git a/src/where.c b/src/where.c index bffca3413d..4ff2815ba8 100644 --- a/src/where.c +++ b/src/where.c @@ -2973,7 +2973,10 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nBtm==0 ); opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; } - if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + if( pProbe->bUnordered || pProbe->bLowQual ){ + if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS); + } assert( pNew->u.btree.nEqnColumn ); assert( pNew->u.btree.nEqnKeyCol From 4cfc19f34c6613061660916119d182fd1cc08309 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 1 Jan 2024 17:58:57 +0000 Subject: [PATCH 11/24] Remove some unnecessary computations from ANALYZE so that ANALYZE runs with fewer CPU cycles. These changes were spotted while working on the nearby enhanced-stat1 branch. So even if enhanced-stat1 is abandoned, that effort put into it will not have been in vain. FossilOrigin-Name: 5527e8c4abb904b1a438ec1c353d4a960bf82faaf3a2c742af1df7c613850441 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/analyze.c | 16 +++++++++++----- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 065f4dad41..bb98e58ef5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\ssteps\staken\sto\savoid\susing\slow-quality\sindexes\sin\sa\squery\splan.\nThis\sbranch\saccomplishes\sthe\ssame\send\sas\sthe\snearby\senhanced-stat1\sbranch,\nbut\swith\smuch\sless\schange\sand\shence\sless\srisk. -D 2024-01-01T17:48:02.427 +C Remove\ssome\sunnecessary\scomputations\sfrom\sANALYZE\sso\sthat\sANALYZE\sruns\swith\nfewer\sCPU\scycles.\s\sThese\schanges\swere\sspotted\swhile\sworking\son\sthe\snearby\nenhanced-stat1\sbranch.\s\sSo\seven\sif\senhanced-stat1\sis\sabandoned,\sthat\seffort\nput\sinto\sit\swill\snot\shave\sbeen\sin\svain. +D 2024-01-01T17:58:57.875 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 30c2333b8bb3af71e4eb9adeadee8aa20edb15917ed44b8422e5cd15f3dfcddc -F src/analyze.c 64ac1dd3b7044c7a7981a5f6d3796bc007b7e31514c674283bf440586681501e +F src/analyze.c 0f15753308c3bca7674f31fa7e0807ffcb8b120c36eef7d00b62b33079ddc854 F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 @@ -2156,11 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc501275fcfab3ad9b6ebbadf7588b225a9dd07a0abac5be83d96f15bfba99e9 -R 4e0f7b3b61e72c244a187959e2e9f486 -T *branch * avoid-low-quality-indexes -T *sym-avoid-low-quality-indexes * -T -sym-trunk * +P c030e646262fee43a59b45fdc1630d972f8bf88ac3c142b6bdaf4cbb36695a4f +R a1cdb8f6aed6a59edfbfbd0410b4be3d U drh -Z ac10d23b30667bc4467501c1b744109a +Z fe0456d193300bdd1fb2616e0f9641df # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c5640b9f7f..ec42b55928 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c030e646262fee43a59b45fdc1630d972f8bf88ac3c142b6bdaf4cbb36695a4f \ No newline at end of file +5527e8c4abb904b1a438ec1c353d4a960bf82faaf3a2c742af1df7c613850441 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index a07f1939c7..59e3d98377 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -264,9 +264,9 @@ static void openStatTable( typedef struct StatAccum StatAccum; typedef struct StatSample StatSample; struct StatSample { - tRowcnt *anEq; /* sqlite_stat4.nEq */ tRowcnt *anDLt; /* sqlite_stat4.nDLt */ #ifdef SQLITE_ENABLE_STAT4 + tRowcnt *anEq; /* sqlite_stat4.nEq */ tRowcnt *anLt; /* sqlite_stat4.nLt */ union { i64 iRowid; /* Rowid in main table of the key */ @@ -424,9 +424,9 @@ static void statInit( /* Allocate the space required for the StatAccum object */ n = sizeof(*p) - + sizeof(tRowcnt)*nColUp /* StatAccum.anEq */ - + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ + + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ #ifdef SQLITE_ENABLE_STAT4 + n += sizeof(tRowcnt)*nColUp; /* StatAccum.anEq */ if( mxSample ){ n += sizeof(tRowcnt)*nColUp /* StatAccum.anLt */ + sizeof(StatSample)*(nCol+mxSample) /* StatAccum.aBest[], a[] */ @@ -447,9 +447,9 @@ static void statInit( p->nKeyCol = nKeyCol; p->nSkipAhead = 0; p->current.anDLt = (tRowcnt*)&p[1]; - p->current.anEq = &p->current.anDLt[nColUp]; #ifdef SQLITE_ENABLE_STAT4 + p->current.anEq = &p->current.anDLt[nColUp]; p->mxSample = p->nLimit==0 ? mxSample : 0; if( mxSample ){ u8 *pSpace; /* Allocated space not yet assigned */ @@ -716,7 +716,9 @@ static void statPush( if( p->nRow==0 ){ /* This is the first call to this function. Do initialization. */ +#ifdef SQLITE_ENABLE_STAT4 for(i=0; inCol; i++) p->current.anEq[i] = 1; +#endif }else{ /* Second and subsequent calls get processed here */ #ifdef SQLITE_ENABLE_STAT4 @@ -725,15 +727,17 @@ static void statPush( /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply ** to the current row of the index. */ +#ifdef SQLITE_ENABLE_STAT4 for(i=0; icurrent.anEq[i]++; } +#endif for(i=iChng; inCol; i++){ p->current.anDLt[i]++; #ifdef SQLITE_ENABLE_STAT4 if( p->mxSample ) p->current.anLt[i] += p->current.anEq[i]; -#endif p->current.anEq[i] = 1; +#endif } } @@ -867,7 +871,9 @@ static void statGet( u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; if( iVal==2 && p->nRow*10 <= nDistinct*11 ) iVal = 1; sqlite3_str_appendf(&sStat, " %llu", iVal); +#ifdef SQLITE_ENABLE_STAT4 assert( p->current.anEq[i] ); +#endif } sqlite3ResultStrAccum(context, &sStat); } From fe1d8210c74f2c2b29867a0e1af7657fb2178d25 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 1 Jan 2024 23:28:02 +0000 Subject: [PATCH 12/24] Back out [99d11e6d0ae6] (enabling of STAT4 in WASM/JNI), per /chat discussion. FossilOrigin-Name: cd7929ee2e2c305475fa5a4dff2edaccf90067126ef04a1c2714cf464925453f --- ext/jni/src/c/sqlite3-jni.c | 3 --- ext/wasm/api/sqlite3-wasm.c | 3 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index 64a2bef198..5deff19ef1 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -76,9 +76,6 @@ //#ifndef SQLITE_ENABLE_SESSION //# define SQLITE_ENABLE_SESSION 1 //#endif -#ifndef SQLITE_ENABLE_STAT4 -# define SQLITE_ENABLE_STAT4 1 -#endif #ifndef SQLITE_ENABLE_STMTVTAB # define SQLITE_ENABLE_STMTVTAB 1 #endif diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 23386c7c70..3e22c799c4 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -122,9 +122,6 @@ #ifndef SQLITE_ENABLE_SESSION # define SQLITE_ENABLE_SESSION 1 #endif -#ifndef SQLITE_ENABLE_STAT4 -# define SQLITE_ENABLE_STAT4 1 -#endif #ifndef SQLITE_ENABLE_STMTVTAB # define SQLITE_ENABLE_STMTVTAB 1 #endif diff --git a/manifest b/manifest index e29e682720..70fb81c936 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\slogic\sto\savoid\susing\sindexes\sthat\sANALYZE\shas\sidentified\sas\sof\slittle\npractical\suse.\s\sAlso\sa\sperformance\soptimization\sin\sANALYZE. -D 2024-01-01T19:20:00.840 +C Back\sout\s[99d11e6d0ae6]\s(enabling\sof\sSTAT4\sin\sWASM/JNI),\sper\s/chat\sdiscussion. +D 2024-01-01T23:28:02.565 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -251,7 +251,7 @@ F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a3 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa -F ext/jni/src/c/sqlite3-jni.c a477b2d0216550cdd6f248c92e4fd666ec2177a350a67417e0b5b9840f52e1c5 +F ext/jni/src/c/sqlite3-jni.c c1292e690a20c7787a63e8d8ac6e2dfed49c97282ed056a7cfda5da461f0b7d8 F ext/jni/src/c/sqlite3-jni.h 913ab8e8fee432ae40f0e387c8231118d17053714703f5ded18202912a8a3fbf F ext/jni/src/org/sqlite/jni/annotation/Experimental.java 8603498634e41d0f7c70f661f64e05df64376562ea8f126829fd1e0cdd47e82b F ext/jni/src/org/sqlite/jni/annotation/NotNull.java 38e7e58a69b26dc100e458b31dfa3b2a7d67bc36d051325526ef1987d5bc8a24 @@ -602,7 +602,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 595953994aa3ae2287c889c4da39ab3d6f17b6461ecf4bec334b7a3faafddb02 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91ccfafc38b400fe36e90bdda88e -F ext/wasm/api/sqlite3-wasm.c dbaefda7981050bf9eb413f5b24df648ead09d9a3e796f9ba84d87cb49b5ab80 +F ext/wasm/api/sqlite3-wasm.c f280d4ea917d213ae95668dfcd173a2c2ef21a0a4bf9aeb9fcd0edaf1b21ba4b F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js f234191fe6bf41a5a1e59c9f43ed816e74a522b3d60d3f556f66c3085c448503 F ext/wasm/api/sqlite3-worker1.c-pp.js 5e8706c2c4af2a57fbcdc02f4e7ef79869971bc21bb8ede777687786ce1c92d5 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 @@ -2156,9 +2156,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc501275fcfab3ad9b6ebbadf7588b225a9dd07a0abac5be83d96f15bfba99e9 5527e8c4abb904b1a438ec1c353d4a960bf82faaf3a2c742af1df7c613850441 -R a1cdb8f6aed6a59edfbfbd0410b4be3d -T +closed 5527e8c4abb904b1a438ec1c353d4a960bf82faaf3a2c742af1df7c613850441 -U drh -Z c24671fca6074048f0170b6af71946ea +P bcac937526d9a6ef914a74b4d6757fa91cd74edab871bcd934fde4a2f9b6debd +Q -99d11e6d0ae687ff6bac5119027f7b04d5e7185214e79cf8c56289cfa809b0f9 +R 3bfcbade7baedb397a7149e377c6dcbb +U stephan +Z 7c711e37eafa39916defd330fece0ca3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 065eb7a539..bf4d9238f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bcac937526d9a6ef914a74b4d6757fa91cd74edab871bcd934fde4a2f9b6debd \ No newline at end of file +cd7929ee2e2c305475fa5a4dff2edaccf90067126ef04a1c2714cf464925453f \ No newline at end of file From 3d4d3fb59dbd565534f32442ef8292ad130d4163 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 2 Jan 2024 09:03:42 +0000 Subject: [PATCH 13/24] Update and clean up the in-makefile docs for ext/wasm. FossilOrigin-Name: 7a7b295e6d7e95ee4a46cc42761895d11700ab295870c5a4380072bb4a5b7099 --- ext/wasm/GNUmakefile | 105 +++++++++++++++++++++++++++--------- ext/wasm/api/sqlite3-wasm.c | 2 +- manifest | 15 +++--- manifest.uuid | 2 +- 4 files changed, 88 insertions(+), 36 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 94eff4cc32..93993ebc47 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -42,7 +42,9 @@ # 1) Consolidate the code generation for sqlite3*.*js into a script # which generates the makefile code, rather than using $(call) and # $(eval), or at least centralize the setup of the numerous vars -# related to each build variant $(JS_BUILD_MODES). +# related to each build variant $(JS_BUILD_MODES). (Update: an +# external script was attempted but it's even less legible than the +# $(eval) indirection going on in this file. # default: all #default: quick @@ -51,12 +53,26 @@ MAKEFILE := $(lastword $(MAKEFILE_LIST)) CLEAN_FILES := DISTCLEAN_FILES := ./--dummy-- release: oz -# JS_BUILD_MODES exists solely to reduce repetition in documentation -# below. -JS_BUILD_MODES := vanilla esm bunder-friendly node + +######################################################################## # JS_BUILD_NAMES exists solely to reduce repetition in documentation -# below. +# below. It enumerates the core build styles: +# +# - sqlite3 = main build +# - sqlite3-wasmfs = WASMFS-capable build JS_BUILD_NAMES := sqlite3 sqlite3-wasmfs + +######################################################################## +# JS_BUILD_MODES exists solely to reduce repetition in documentation +# below. It enumerates the various flavors of build: +# +# - vanilla = plain-vanilla JS +# - ecm = ES6 module a.k.a. ESM +# - bundler-friendly = esm slightly tweaked for "bundler" tools +# - node = for use with node.js +JS_BUILD_MODES := vanilla esm bunder-friendly node + +######################################################################## # Emscripten SDK home dir and related binaries... EMSDK_HOME ?= $(word 1,$(wildcard $(HOME)/emsdk $(HOME)/src/emsdk)) emcc.bin ?= $(word 1,$(wildcard $(EMSDK_HOME)/upstream/emscripten/emcc) $(shell which emcc)) @@ -96,11 +112,14 @@ else maybe-wasm-strip = $(wasm-strip) endif +######################################################################## +# dir.top = the top dir of the canonical build tree, where +# sqlite3.[ch] live. dir.top := ../.. -# Reminder: some Emscripten flags require absolute paths but we want -# relative paths for most stuff simply to reduce noise. The -# $(abspath...) GNU make function can transform relative paths to -# absolute. +# Maintenance reminder: some Emscripten flags require absolute paths +# but we want relative paths for most stuff simply to reduce +# noise. The $(abspath...) GNU make function can transform relative +# paths to absolute. dir.wasm := $(patsubst %/,%,$(dir $(MAKEFILE))) dir.api := api dir.jacc := jaccwabyt @@ -146,12 +165,14 @@ endif # Set up sqlite3.c and sqlite3.h... # # To build with SEE (https://sqlite.org/see), either put sqlite3-see.c -# in the top of this build tree or pass -# sqlite3.c=PATH_TO_sqlite3-see.c to the build. Note that only -# encryption modules with no 3rd-party dependencies will currently -# work here: AES256-OFB, AES128-OFB, and AES128-CCM. Not -# coincidentally, those 3 modules are included in the sqlite3-see.c -# bundle. +# in $(dir.top) or pass sqlite3.c=PATH_TO_sqlite3-see.c to the $(MAKE) +# invocation. Note that only encryption modules with no 3rd-party +# dependencies will currently work here: AES256-OFB, AES128-OFB, and +# AES128-CCM. Not coincidentally, those 3 modules are included in the +# sqlite3-see.c bundle. Note, however, that distributing an SEE build +# of the WASM on a public site is in violation of the SEE license +# because it effectively provides a usable copy of the SEE build to +# all visitors. # # A custom sqlite3.c must not have any spaces in its name. # $(sqlite3.canonical.c) must point to the sqlite3.c in @@ -196,6 +217,10 @@ SQLITE_OPT = \ # can be used to find errant uses of sqlite3_js_vfs_create_file() # in client code. +########################################################################@ +# It's important that sqlite3.h be built to completion before any +# other parts of the build run, thus we use .NOTPARALLEL to disable +# parallel build of that file and its dependants. .NOTPARALLEL: $(sqlite3.h) $(sqlite3.h): $(MAKE) -C $(dir.top) sqlite3.c @@ -245,6 +270,7 @@ ifneq (,$(sqlite3_wasm_extra_init.c)) cflags.wasm_extra_init := -DSQLITE_WASM_EXTRA_INIT endif +######################################################################### # bin.version-info = binary to output various sqlite3 version info for # embedding in the JS files and in building the distribution zip file. # It must NOT be in $(dir.tmp) because we need it to survive the @@ -254,11 +280,12 @@ bin.version-info := $(dir.top)/version-info $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile $(MAKE) -C $(dir.top) version-info +######################################################################### # bin.stripcomments is used for stripping C/C++-style comments from JS # files. The JS files contain large chunks of documentation which we # don't need for all builds. That app's -k flag is of particular # importance here, as it allows us to retain the opening comment -# blocks, which contain the license header and version info. +# block(s), which contain the license header and version info. bin.stripccomments := $(dir.tool)/stripccomments $(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) $(CC) -o $@ $< @@ -353,6 +380,7 @@ emcc_opt_full := $(emcc_opt) -g3 # -Oz when small deliverable size is a priority. ######################################################################## +######################################################################## # EXPORTED_FUNCTIONS.* = files for use with Emscripten's # -sEXPORTED_FUNCTION flag. EXPORTED_FUNCTIONS.api.main := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) @@ -364,6 +392,7 @@ EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api $(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) cat $(EXPORTED_FUNCTIONS.api.in) > $@ +######################################################################## # sqlite3-license-version.js = generated JS file with the license # header and version info. sqlite3-license-version.js := $(dir.tmp)/sqlite3-license-version.js @@ -376,20 +405,35 @@ sqlite3-api-build-version.js := $(dir.tmp)/sqlite3-api-build-version.js # sqlite3-api.jses = the list of JS files which make up # $(sqlite3-api.js.in), in the order they need to be assembled. sqlite3-api.jses := $(sqlite3-license-version.js) +# sqlite3-api-prologue.js: initial boostrapping bits: sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js +# whwhasm.js and jaccwabyt.js: Low-level utils, mostly replacing +# Emscripten glue: sqlite3-api.jses += $(dir.common)/whwasmutil.js sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js +# sqlite3-api-glue.js Glues the previous part together: sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js +# $(sqlite3-api-build-version.js) = library version info sqlite3-api.jses += $(sqlite3-api-build-version.js) +# sqlite3-api-oo1.js = the oo1 API: sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js +# sqlite3-api-worker.js = the Worker1 API: sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js +# sqlite3-v-helper = helper APIs for VFSes and VTABLEs: sqlite3-api.jses += $(dir.api)/sqlite3-v-helper.js +# sqlite3-vfs-opfs.c-pp.js = the first OPFS VFS: sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js +# sqlite3-vfs-opfs-sahpool.c-pp.js = the second OPFS VFS: sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js +# sqlite3-api-cleanup.js = "finalizes" the build and cleans up +# any extraneous global symbols which are needed temporarily +# by the previous files. sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js +######################################################################## # SOAP.js is an external API file which is part of our distribution -# but not part of the sqlite3-api.js amalgamation. +# but not part of the sqlite3-api.js amalgamation. It's a component of +# the first OPFS VFS and necessarily an external file. SOAP.js := $(dir.api)/sqlite3-opfs-async-proxy.js SOAP.js.bld := $(dir.dout)/$(notdir $(SOAP.js)) sqlite3-api.ext.jses += $(SOAP.js.bld) @@ -444,7 +488,9 @@ endif # emcc flags for .c/.o. emcc.cflags := emcc.cflags += -std=c99 -fPIC -# -------------^^^^^^^^ we need c99 for $(sqlite3-wasm.c). +# -------------^^^^^^^^ we need c99 for $(sqlite3-wasm.c), primarily +# for variadic macros and snprintf() to implement +# sqlite3_wasm_enum_json(). emcc.cflags += -I. -I$(dir.top) ######################################################################## # emcc flags specific to building .js/.wasm files... @@ -465,14 +511,16 @@ emcc.jsflags += -sIMPORTED_MEMORY emcc.jsflags += -sSTRICT_JS=0 # STRICT_JS disabled due to: # https://github.com/emscripten-core/emscripten/issues/18610 -# TL;DR: does not work with MODULARIZE or EXPORT_ES6 as of version 3.1.31. +# TL;DR: does not work with MODULARIZE or EXPORT_ES6 as of version +# 3.1.31. The fix for that in newer emcc's is to throw a built-time +# error if STRICT_JS is used together with those options. # -sENVIRONMENT values for the various build modes: emcc.environment.vanilla := web,worker emcc.environment.bundler-friendly := $(emcc.environment.vanilla) emcc.environment.esm := $(emcc.environment.vanilla) emcc.environment.node := node -# Note that adding "node" to the list for the other builds causes +# Note that adding ",node" to the list for the other builds causes # Emscripten to generate code which confuses node: it cannot reliably # determine whether the build is for a browser or for node. @@ -575,8 +623,9 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED # -g3 debugging info, _huge_. ######################################################################## +######################################################################## # $(sqlite3-api-build-version.js) injects the build version info into -# the bundle. +# the bundle in JSON form. $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) @echo "Making $@..." @{ \ @@ -587,8 +636,9 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) echo '});'; \ } > $@ -# $(sqlite3-license-version.js) contains the license header and build -# version info. +######################################################################## +# $(sqlite3-license-version.js) contains the license header and +# in-comment build version info. $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) \ $(MAKEFILE) @echo "Making $@..."; { \ @@ -707,8 +757,8 @@ sqlite3-wasmfs.cfiles := $(sqlite3-wasm.cfiles) # Upstream RFE: # https://github.com/emscripten-core/emscripten/issues/18237 # -# Maintenance reminder: Mac sed works differently than GNU sed, so -# don't use sed for this. +# Maintenance reminder: Mac sed works differently than GNU sed, so we +# use awk instead of sed for this. define SQLITE3.xJS.ESM-EXPORT-DEFAULT if [ x1 = x$(1) ]; then \ echo "Fragile workaround for emscripten/issues/18237. See SQLITE3.xJS.RECIPE."; \ @@ -724,6 +774,7 @@ if [ x1 = x$(1) ]; then \ fi endef +######################################################################## # extern-post-js* and extern-pre-js* are files for use with # Emscripten's --extern-pre-js and --extern-post-js flags. extern-pre-js.js := $(dir.api)/extern-pre-js.js @@ -735,6 +786,7 @@ pre-post-common.flags := \ # pre-post-jses.deps.* = a list of dependencies for the # --[extern-][pre/post]-js files. pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) + ######################################################################## # SETUP_LIB_BUILD_MODE is a $(call)'able which sets up numerous pieces # for one of the build modes. @@ -750,7 +802,8 @@ pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) # Maintenance reminder: be careful not to introduce spaces around args # ($1, $2), otherwise string concatenation will malfunction. # -# emcc.environment.$(2) must be set to a value for the -sENVIRONMENT flag. +# emcc.environment.$(2) must be set to a value for emcc's +# -sENVIRONMENT flag. # # $(cflags.$(1)) and $(cflags.$(1).$(2)) may be defined to append # CFLAGS to a given build mode. diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 3e22c799c4..618d0f085a 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -1103,7 +1103,7 @@ const char * sqlite3_wasm_enum_json(void){ M(xShadowName, "i(s)"); } _StructBinder; #undef CurrentStruct - + /** ** Workaround: in order to map the various inner structs from ** sqlite3_index_info, we have to uplift those into constructs we diff --git a/manifest b/manifest index 70fb81c936..50fd1af7d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Back\sout\s[99d11e6d0ae6]\s(enabling\sof\sSTAT4\sin\sWASM/JNI),\sper\s/chat\sdiscussion. -D 2024-01-01T23:28:02.565 +C Update\sand\sclean\sup\sthe\sin-makefile\sdocs\sfor\sext/wasm. +D 2024-01-02T09:03:42.100 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile a8a171af359b9acee604bb01229f59052b8b38c43f6020a6d009e42c76f910af +F ext/wasm/GNUmakefile 5b1f425d3619578d4e17312a3d920cbf0d22caff67d1fc30330046ca0ae24bc6 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -602,7 +602,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 595953994aa3ae2287c889c4da39ab3d6f17b6461ecf4bec334b7a3faafddb02 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91ccfafc38b400fe36e90bdda88e -F ext/wasm/api/sqlite3-wasm.c f280d4ea917d213ae95668dfcd173a2c2ef21a0a4bf9aeb9fcd0edaf1b21ba4b +F ext/wasm/api/sqlite3-wasm.c dfd1f1a225b267e8fd641dcd6c7d579fbe2b731aeaa123324135efac830a2bcf F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js f234191fe6bf41a5a1e59c9f43ed816e74a522b3d60d3f556f66c3085c448503 F ext/wasm/api/sqlite3-worker1.c-pp.js 5e8706c2c4af2a57fbcdc02f4e7ef79869971bc21bb8ede777687786ce1c92d5 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 @@ -2156,9 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bcac937526d9a6ef914a74b4d6757fa91cd74edab871bcd934fde4a2f9b6debd -Q -99d11e6d0ae687ff6bac5119027f7b04d5e7185214e79cf8c56289cfa809b0f9 -R 3bfcbade7baedb397a7149e377c6dcbb +P cd7929ee2e2c305475fa5a4dff2edaccf90067126ef04a1c2714cf464925453f +R de6e93274e75f378c4f5110a14f6c9e3 U stephan -Z 7c711e37eafa39916defd330fece0ca3 +Z 326fdaaa6c149e5a91868eade9fb0075 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bf4d9238f0..28b1ac3a3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd7929ee2e2c305475fa5a4dff2edaccf90067126ef04a1c2714cf464925453f \ No newline at end of file +7a7b295e6d7e95ee4a46cc42761895d11700ab295870c5a4380072bb4a5b7099 \ No newline at end of file From aff766493938a89f6fc0146ab898c64ec8836a8f Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 2 Jan 2024 09:20:53 +0000 Subject: [PATCH 14/24] Elaborate on the various build flavors used by ext/wasm/. Doc changes only. FossilOrigin-Name: d489232aa492618d4c8e5817addb2323d0ca067742d7140216914239a66fb221 --- ext/wasm/GNUmakefile | 35 ++++++++++++++++++++++++----------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 93993ebc47..f0cff463a8 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -55,21 +55,34 @@ DISTCLEAN_FILES := ./--dummy-- release: oz ######################################################################## -# JS_BUILD_NAMES exists solely to reduce repetition in documentation -# below. It enumerates the core build styles: +# JS_BUILD_NAMES exists for documentation purposes only. It enumerates +# the core build styles: +# +# - sqlite3 = canonical library build +# +# - sqlite3-wasmfs = WASMFS-capable library build # -# - sqlite3 = main build -# - sqlite3-wasmfs = WASMFS-capable build JS_BUILD_NAMES := sqlite3 sqlite3-wasmfs ######################################################################## -# JS_BUILD_MODES exists solely to reduce repetition in documentation -# below. It enumerates the various flavors of build: +# JS_BUILD_MODES exists for documentation purposes only. It enumerates +# the various "flavors" of build, each of which requires slight +# customization of the output: +# +# - vanilla = plain-vanilla JS for use in browsers. This is the +# canonical build mode. +# +# - esm = ES6 module, a.k.a. ESM, for use in browsers. +# +# - bundler-friendly = esm slightly tweaked for "bundler" +# tools. Bundlers are invariably based on node.js, so these builds +# are intended to be read at build-time by node.js but with a final +# target of browsers. +# +# - node = for use by node.js for node.js, as opposed to by node.js on +# behalf o browser-side code (use bundler-friendly for that). Note +# that persistent storage (OPFS) is not available in these builds. # -# - vanilla = plain-vanilla JS -# - ecm = ES6 module a.k.a. ESM -# - bundler-friendly = esm slightly tweaked for "bundler" tools -# - node = for use with node.js JS_BUILD_MODES := vanilla esm bunder-friendly node ######################################################################## @@ -874,7 +887,7 @@ $(eval $(call SETUP_LIB_BUILD_MODE,sqlite3,node,1,\ # -Dtarget=es6-module -Dtarget=es6-bundler-friendly: intended for # "bundler-friendly" ESM module build. These have some restrictions # on how URL() objects are constructed in some contexts: URLs which -# refer to files which are part of this project must be references +# refer to files which are part of this project must be referenced # as string literals so that bundlers' static-analysis tools can # find those files and include them in their bundles. # diff --git a/manifest b/manifest index 50fd1af7d4..9b1ae910a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sand\sclean\sup\sthe\sin-makefile\sdocs\sfor\sext/wasm. -D 2024-01-02T09:03:42.100 +C Elaborate\son\sthe\svarious\sbuild\sflavors\sused\sby\sext/wasm/.\sDoc\schanges\sonly. +D 2024-01-02T09:20:53.920 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 5b1f425d3619578d4e17312a3d920cbf0d22caff67d1fc30330046ca0ae24bc6 +F ext/wasm/GNUmakefile 99aad6d6a28c43573f80825e986427c1a024a3298aaf0c69c56a0c6b336f12c8 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cd7929ee2e2c305475fa5a4dff2edaccf90067126ef04a1c2714cf464925453f -R de6e93274e75f378c4f5110a14f6c9e3 +P 7a7b295e6d7e95ee4a46cc42761895d11700ab295870c5a4380072bb4a5b7099 +R dbfbc413c8ad5978806e35b24ed8829e U stephan -Z 326fdaaa6c149e5a91868eade9fb0075 +Z 4ad8f4d46176eebccf80225f45577976 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 28b1ac3a3f..85019a5870 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a7b295e6d7e95ee4a46cc42761895d11700ab295870c5a4380072bb4a5b7099 \ No newline at end of file +d489232aa492618d4c8e5817addb2323d0ca067742d7140216914239a66fb221 \ No newline at end of file From 533622fb8b6f2431f772e4ec9f98aca27e354ca0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 2 Jan 2024 20:34:27 +0000 Subject: [PATCH 15/24] Increase the default "max_page_count" to its theoretical maximum of 4294967294. FossilOrigin-Name: ffb35f1784a4305b979a850485f57f56938104a3a03f4a7aececde92864c4879 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteLimit.h | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9b1ae910a9..c5dc2aed32 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Elaborate\son\sthe\svarious\sbuild\sflavors\sused\sby\sext/wasm/.\sDoc\schanges\sonly. -D 2024-01-02T09:20:53.920 +C Increase\sthe\sdefault\s"max_page_count"\sto\sits\stheoretical\smaximum\sof\n4294967294. +D 2024-01-02T20:34:27.723 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -743,7 +743,7 @@ F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 F src/sqliteInt.h 73800d73e21180e6b3df8d0fe7d11758dc24367fd2b0b0075b48fc116de406bb -F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 +F src/sqliteLimit.h 0c4b40bfebb193b5a4eb13db928413e618be8b05fcb572ddf03664c036568f7f F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c ecbc3c99c0d0c3ed122a913f143026c26d38d57f33e06bb71185dd5c1efe37cd @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7a7b295e6d7e95ee4a46cc42761895d11700ab295870c5a4380072bb4a5b7099 -R dbfbc413c8ad5978806e35b24ed8829e -U stephan -Z 4ad8f4d46176eebccf80225f45577976 +P d489232aa492618d4c8e5817addb2323d0ca067742d7140216914239a66fb221 +R a0497c811eff87495d0c9ffce0ab9884 +U drh +Z d3582aa92c333af17186884e8cd6645f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 85019a5870..0a0bd94825 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d489232aa492618d4c8e5817addb2323d0ca067742d7140216914239a66fb221 \ No newline at end of file +ffb35f1784a4305b979a850485f57f56938104a3a03f4a7aececde92864c4879 \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index bfb596da23..0eaadec1cb 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -187,7 +187,7 @@ ** max_page_count macro. */ #ifndef SQLITE_MAX_PAGE_COUNT -# define SQLITE_MAX_PAGE_COUNT 1073741823 +# define SQLITE_MAX_PAGE_COUNT 4294967294 #endif /* From 4eb626350a8359671fa62712be5b33d4a605d444 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Jan 2024 21:08:25 +0000 Subject: [PATCH 16/24] Fix a problem in fts5 caused by a COMMIT involving fts5 data that immediately follows a ROLLBACK TO that does not. FossilOrigin-Name: 55c61f6a8d6a1bc79497b05669beac5c5397b06382bf24b6bec54845962d219b --- ext/fts5/fts5_main.c | 2 +- ext/fts5/test/fts5misc.test | 20 ++++++++++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 05946d8b64..45dee44b1e 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2731,8 +2731,8 @@ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ int rc = SQLITE_OK; fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); - pTab->p.pConfig->pgsz = 0; if( (iSavepoint+1)<=pTab->iSavepoint ){ + pTab->p.pConfig->pgsz = 0; rc = sqlite3Fts5StorageRollback(pTab->pStorage); } return rc; diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index 221f3996a7..4e4fe4ba2d 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -515,5 +515,25 @@ do_execsql_test 18.2 { COMMIT; } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 19.0 { + CREATE VIRTUAL TABLE t1 USING fts5(text); + CREATE TABLE t2(text); + BEGIN; + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('two'); + INSERT INTO t1 VALUES('three'); + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('two'); + INSERT INTO t1 VALUES('three'); + SAVEPOINT one; + INSERT INTO t2 VALUES('one'); + INSERT INTO t2 VALUES('two'); + INSERT INTO t2 VALUES('three'); + ROLLBACK TO one; + COMMIT; +} + finish_test diff --git a/manifest b/manifest index c5dc2aed32..c374d37ee2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sdefault\s"max_page_count"\sto\sits\stheoretical\smaximum\sof\n4294967294. -D 2024-01-02T20:34:27.723 +C Fix\sa\sproblem\sin\sfts5\scaused\sby\sa\sCOMMIT\sinvolving\sfts5\sdata\sthat\simmediately\sfollows\sa\sROLLBACK\sTO\sthat\sdoes\snot. +D 2024-01-02T21:08:25.050 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -98,7 +98,7 @@ F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532 F ext/fts5/fts5_expr.c e91156ebdcc08d837f4f324168f69f3c0d7fdef0e521fd561efb48ef3297b696 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c bb1965c3965f6fe5f64160bf1c0694a9684a790a783f293a76da1d38d319b258 -F ext/fts5/fts5_main.c 93e5bc8676dbaaec365576d26cdafc5c21f0052f8f68b86ee4de22237c2e2143 +F ext/fts5/fts5_main.c 1c6c7df07fd0d8b37548887dcdc00a8cc31fd1dc13bf9091f3350a9f23e6353c F ext/fts5/fts5_storage.c f9e31b0d155e9b2c92d5d3a09ad7a56b937fbf1c7f962e10f4ca6281349f3934 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -184,7 +184,7 @@ F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fc F ext/fts5/test/fts5matchinfo.test 10c9a6f7fe61fb132299c4183c012770b10c4d5c2f2edb6df0b6607f683d737a F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test dd97c86c9cbc3e587067e640f6ce88842cfbf5d23bb0e0fbb7f6707623b2d505 +F ext/fts5/test/fts5misc.test 89dc46e37951b7f6653809f4abf6b1ca2f1fa62259efaf719339288f76fb6be9 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d489232aa492618d4c8e5817addb2323d0ca067742d7140216914239a66fb221 -R a0497c811eff87495d0c9ffce0ab9884 -U drh -Z d3582aa92c333af17186884e8cd6645f +P ffb35f1784a4305b979a850485f57f56938104a3a03f4a7aececde92864c4879 +R e7196a3c488f0fb11fc6e934ceab1b33 +U dan +Z 8628e9692564183c206ecaf3e3550685 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0a0bd94825..62069135fb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffb35f1784a4305b979a850485f57f56938104a3a03f4a7aececde92864c4879 \ No newline at end of file +55c61f6a8d6a1bc79497b05669beac5c5397b06382bf24b6bec54845962d219b \ No newline at end of file From f5740f72ecc3e52da9dc6321dd3361c65f1125ca Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 2 Jan 2024 21:37:51 +0000 Subject: [PATCH 17/24] Adjust the sqlite3PagerDirectReadOk() routine (part of the SQLITE_DIRECT_OVERFLOW_READ optimization) to use less code and to be more easily testable. FossilOrigin-Name: eed670ea2a9424f7df4eeb01c152fc38f7190a5e39aa891651b28dc91fcdc019 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 5 ++--- src/wal.c | 1 + 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c374d37ee2..9e60870371 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sfts5\scaused\sby\sa\sCOMMIT\sinvolving\sfts5\sdata\sthat\simmediately\sfollows\sa\sROLLBACK\sTO\sthat\sdoes\snot. -D 2024-01-02T21:08:25.050 +C Adjust\sthe\ssqlite3PagerDirectReadOk()\sroutine\s(part\sof\sthe\nSQLITE_DIRECT_OVERFLOW_READ\soptimization)\sto\suse\sless\scode\sand\sto\sbe\nmore\seasily\stestable. +D 2024-01-02T21:37:51.081 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -724,7 +724,7 @@ F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d87210 F src/os_unix.c 1672c708df279fca1b6ba619cbb26a88baa7913b21dda95817290d76666a9688 F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c fef16beb072d934ca4f3ef496b2b65703fe2291c4b858782a94cf031c626e816 +F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a F src/parse.y 020d80386eb216ec9520549106353c517d2bbc89be28752ffdca649a9eaf56ec F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 @@ -819,7 +819,7 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8 F src/vdbevtab.c 2143db7db0ceed69b21422581f434baffc507a08d831565193a7a02882a1b6a7 F src/vtab.c 11948e105f56e84099ca17f1f434b1944539ea84de26d0d767eadfbc670ce1ea F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20 +F src/wal.c b08f88e69b9c43572f0703355dccc5b2f08296256fc6a82a582fcae68f8f9cd1 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 F src/where.c 217fe82a26c0fb6a3c7fd01865d821e752f9c01fb72f114af3f0b77ce234d1fb @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ffb35f1784a4305b979a850485f57f56938104a3a03f4a7aececde92864c4879 -R e7196a3c488f0fb11fc6e934ceab1b33 -U dan -Z 8628e9692564183c206ecaf3e3550685 +P 55c61f6a8d6a1bc79497b05669beac5c5397b06382bf24b6bec54845962d219b +R 867bf5cb3c0877715a9ed5af28923efb +U drh +Z 4daa50f8a69fc469a847112ed02f070f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 62069135fb..a733dc90fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55c61f6a8d6a1bc79497b05669beac5c5397b06382bf24b6bec54845962d219b \ No newline at end of file +eed670ea2a9424f7df4eeb01c152fc38f7190a5e39aa891651b28dc91fcdc019 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index f779d74573..37588f0b25 100644 --- a/src/pager.c +++ b/src/pager.c @@ -818,9 +818,8 @@ int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ #ifndef SQLITE_OMIT_WAL if( pPager->pWal ){ u32 iRead = 0; - int rc; - rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); - return (rc==SQLITE_OK && iRead==0); + (void)sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + return iRead==0; } #endif return 1; diff --git a/src/wal.c b/src/wal.c index fa22e64c84..eb97831dcb 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3505,6 +3505,7 @@ static int walFindFrame( iRead = iFrame; } if( (nCollide--)==0 ){ + *piRead = 0; return SQLITE_CORRUPT_BKPT; } iKey = walNextHash(iKey); From d059153deb6154108ce71d557f8d50e173735267 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 2 Jan 2024 22:49:14 +0000 Subject: [PATCH 18/24] Back out [b517a52fa36df0a0] which is no longer reachable due to early error detection enhancements in [166e82dd20efbfd3]. FossilOrigin-Name: 704943e96f2620b99260667ac9922c2f72bc3e92e2dfe1d9c2a91c7b704564d9 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/btree.c | 1 - 3 files changed, 8 insertions(+), 8 deletions(-) mode change 100755 => 100644 src/btree.c diff --git a/manifest b/manifest index 9e60870371..f462fd8416 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\ssqlite3PagerDirectReadOk()\sroutine\s(part\sof\sthe\nSQLITE_DIRECT_OVERFLOW_READ\soptimization)\sto\suse\sless\scode\sand\sto\sbe\nmore\seasily\stestable. -D 2024-01-02T21:37:51.081 +C Back\sout\s[b517a52fa36df0a0]\swhich\sis\sno\slonger\sreachable\sdue\sto\searly\nerror\sdetection\senhancements\sin\s[166e82dd20efbfd3]. +D 2024-01-02T22:49:14.643 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -676,7 +676,7 @@ F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 -F src/btree.c f3b09c5414de3a11db73e11e1d66f4c5e53c9e89876ff3b531a887ab656ca303 x +F src/btree.c dee25e097b749275333b55d64a5ffc079249576f8e88a2ee476468cf67510f4b F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240 F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062 F src/build.c 386eadecabe2e99a3783eb802ca01e665f8e0c2af0e0aab28161fd7def219a9d @@ -2156,8 +2156,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 55c61f6a8d6a1bc79497b05669beac5c5397b06382bf24b6bec54845962d219b -R 867bf5cb3c0877715a9ed5af28923efb +P eed670ea2a9424f7df4eeb01c152fc38f7190a5e39aa891651b28dc91fcdc019 +Q -b517a52fa36df0a0854a75858b5e81861771d2e9032a5a0ad79aa76ae64130a2 +R 9543e433ed40d6e55dfb715e04bc981d U drh -Z 4daa50f8a69fc469a847112ed02f070f +Z 20a621e221a4c4291644be7f432a4510 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a733dc90fc..66021d274f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eed670ea2a9424f7df4eeb01c152fc38f7190a5e39aa891651b28dc91fcdc019 \ No newline at end of file +704943e96f2620b99260667ac9922c2f72bc3e92e2dfe1d9c2a91c7b704564d9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c old mode 100755 new mode 100644 index 5be30d562c..907e37f1ea --- a/src/btree.c +++ b/src/btree.c @@ -5161,7 +5161,6 @@ static int accessPayload( assert( aWrite>=pBufStart ); /* due to (6) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); - if( rc && nextPage>pBt->nPage ) rc = SQLITE_CORRUPT_BKPT; nextPage = get4byte(aWrite); memcpy(aWrite, aSave, 4); }else From 4167033eaf943ed12ff288eaa1ab9bb2af9327d0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 3 Jan 2024 12:26:02 +0000 Subject: [PATCH 19/24] Update the sqldiff.exe utility program so that it uses the sqlite3_str string interface, and so that it does console output using the ext/consio extension. FossilOrigin-Name: 4443b7e592da97d1cb1b3b79ed0559452d8057a33aba4d184c2fffbf200e05f5 --- Makefile.msc | 4 +- manifest | 15 +- manifest.uuid | 2 +- tool/sqldiff.c | 440 +++++++++++++++++++++++-------------------------- 4 files changed, 220 insertions(+), 241 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 15c44b62f9..d862fbbfb1 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1828,8 +1828,8 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) # <> -sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) - $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) +sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.h $(TOP)\ext\consio\console_io.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) + $(LTLINK) $(NO_WARN) -I$(TOP)\ext\consio $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) diff --git a/manifest b/manifest index f462fd8416..7aa5d0ca33 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Back\sout\s[b517a52fa36df0a0]\swhich\sis\sno\slonger\sreachable\sdue\sto\searly\nerror\sdetection\senhancements\sin\s[166e82dd20efbfd3]. -D 2024-01-02T22:49:14.643 +C Update\sthe\ssqldiff.exe\sutility\sprogram\sso\sthat\sit\suses\sthe\ssqlite3_str\nstring\sinterface,\sand\sso\sthat\sit\sdoes\sconsole\soutput\susing\sthe\next/consio\sextension. +D 2024-01-03T12:26:02.221 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 24be65ae641c5727bbc247d60286a15ecc24fb80f14f8fb2d32533bf0ec96e79 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc ddca98e096947fc26e3bb1a20678999dd0a590456278ad7dc55b882fca108bd6 +F Makefile.msc 8d4860af68385187806e16757bcbca0b852a7ae1f0618fa2eb42fe51166c5bb6 F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3 F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -2117,7 +2117,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 -F tool/sqldiff.c fcccbc07da942b4534d0c769e9fcc21c67cbd7086ddc1c8f13372c40a83d4634 +F tool/sqldiff.c 985452ffc8554baee0f945d078859d393a22abc0bbf553a9f12acae1b6e1fdd0 F tool/sqlite3_analyzer.c.in f88615bf33098945e0a42f17733f472083d150b58bdaaa5555a7129d0a51621c F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2156,9 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eed670ea2a9424f7df4eeb01c152fc38f7190a5e39aa891651b28dc91fcdc019 -Q -b517a52fa36df0a0854a75858b5e81861771d2e9032a5a0ad79aa76ae64130a2 -R 9543e433ed40d6e55dfb715e04bc981d +P 704943e96f2620b99260667ac9922c2f72bc3e92e2dfe1d9c2a91c7b704564d9 +R e56f90f682975bdbaf88cd4673a534d1 U drh -Z 20a621e221a4c4291644be7f432a4510 +Z a3427ef23d574bfd370854aa9641acf2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 66021d274f..00c224a725 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -704943e96f2620b99260667ac9922c2f72bc3e92e2dfe1d9c2a91c7b704564d9 \ No newline at end of file +4443b7e592da97d1cb1b3b79ed0559452d8057a33aba4d184c2fffbf200e05f5 \ No newline at end of file diff --git a/tool/sqldiff.c b/tool/sqldiff.c index a612566112..cbdfc35cd0 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -13,7 +13,8 @@ ** This is a utility program that computes the differences in content ** between two SQLite databases. ** -** To compile, simply link against SQLite. +** To compile, simply link against SQLite. (Windows builds must also link +** against ext/consio/console_io.c.) ** ** See the showHelp() routine below for a brief description of how to ** run the utility. @@ -26,6 +27,19 @@ #include #include "sqlite3.h" +/* Output function substitutions that cause UTF8 characters to be rendered +** correctly on Windows: +** +** fprintf() -> Wfprintf() +** +*/ +#if defined(_WIN32) +# include "console_io.h" +# define Wfprintf fPrintfUtf8 +#else +# define Wfprintf fprintf +#endif + /* ** All global variables are gathered into the "g" singleton. */ @@ -46,22 +60,10 @@ struct GlobalVars { #define DEBUG_DIFF_SQL 0x000002 /* -** Dynamic string object +** Clear and free an sqlite3_str object */ -typedef struct Str Str; -struct Str { - char *z; /* Text of the string */ - int nAlloc; /* Bytes allocated in z[] */ - int nUsed; /* Bytes actually used in z[] */ -}; - -/* -** Initialize a Str object -*/ -static void strInit(Str *p){ - p->z = 0; - p->nAlloc = 0; - p->nUsed = 0; +static void strFree(sqlite3_str *pStr){ + sqlite3_free(sqlite3_str_finish(pStr)); } /* @@ -69,12 +71,14 @@ static void strInit(Str *p){ ** abort the program. */ static void cmdlineError(const char *zFormat, ...){ + sqlite3_str *pOut = sqlite3_str_new(0); va_list ap; - fprintf(stderr, "%s: ", g.zArgv0); va_start(ap, zFormat); - vfprintf(stderr, zFormat, ap); + sqlite3_str_vappendf(pOut, zFormat, ap); va_end(ap); - fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0); + Wfprintf(stderr, "%s: %s\n", g.zArgv0, sqlite3_str_value(pOut)); + strFree(pOut); + Wfprintf(stderr, "\"%s --help\" for more help\n", g.zArgv0); exit(1); } @@ -83,49 +87,16 @@ static void cmdlineError(const char *zFormat, ...){ ** abort the program. */ static void runtimeError(const char *zFormat, ...){ + sqlite3_str *pOut = sqlite3_str_new(0); va_list ap; - fprintf(stderr, "%s: ", g.zArgv0); va_start(ap, zFormat); - vfprintf(stderr, zFormat, ap); + sqlite3_str_vappendf(pOut, zFormat, ap); va_end(ap); - fprintf(stderr, "\n"); + Wfprintf(stderr, "%s: %s\n", g.zArgv0, sqlite3_str_value(pOut)); + strFree(pOut); exit(1); } -/* -** Free all memory held by a Str object -*/ -static void strFree(Str *p){ - sqlite3_free(p->z); - strInit(p); -} - -/* -** Add formatted text to the end of a Str object -*/ -static void strPrintf(Str *p, const char *zFormat, ...){ - int nNew; - for(;;){ - if( p->z ){ - va_list ap; - va_start(ap, zFormat); - sqlite3_vsnprintf(p->nAlloc-p->nUsed, p->z+p->nUsed, zFormat, ap); - va_end(ap); - nNew = (int)strlen(p->z + p->nUsed); - }else{ - nNew = p->nAlloc; - } - if( p->nUsed+nNew < p->nAlloc-1 ){ - p->nUsed += nNew; - break; - } - p->nAlloc = p->nAlloc*2 + 1000; - p->z = sqlite3_realloc(p->z, p->nAlloc); - if( p->z==0 ) runtimeError("out of memory"); - } -} - - /* Safely quote an SQL identifier. Use the minimum amount of transformation ** necessary to allow the string to be used with %s. @@ -453,7 +424,7 @@ static void dump_table(const char *zTab, FILE *out){ int i; /* Loop counter */ sqlite3_stmt *pStmt; /* SQL statement */ const char *zSep; /* Separator string */ - Str ins; /* Beginning of the INSERT statement */ + sqlite3_str *pIns; /* Beginning of the INSERT statement */ pStmt = db_prepare("SELECT sql FROM aux.sqlite_schema WHERE name=%Q", zTab); if( SQLITE_ROW==sqlite3_step(pStmt) ){ @@ -462,54 +433,53 @@ static void dump_table(const char *zTab, FILE *out){ sqlite3_finalize(pStmt); if( !g.bSchemaOnly ){ az = columnNames("aux", zTab, &nPk, 0); - strInit(&ins); + pIns = sqlite3_str_new(0); if( az==0 ){ pStmt = db_prepare("SELECT * FROM aux.%s", zId); - strPrintf(&ins,"INSERT INTO %s VALUES", zId); + sqlite3_str_appendf(pIns,"INSERT INTO %s VALUES", zId); }else{ - Str sql; - strInit(&sql); + sqlite3_str *pSql = sqlite3_str_new(0); zSep = "SELECT"; for(i=0; az[i]; i++){ - strPrintf(&sql, "%s %s", zSep, az[i]); + sqlite3_str_appendf(pSql, "%s %s", zSep, az[i]); zSep = ","; } - strPrintf(&sql," FROM aux.%s", zId); + sqlite3_str_appendf(pSql," FROM aux.%s", zId); zSep = " ORDER BY"; for(i=1; i<=nPk; i++){ - strPrintf(&sql, "%s %d", zSep, i); + sqlite3_str_appendf(pSql, "%s %d", zSep, i); zSep = ","; } - pStmt = db_prepare("%s", sql.z); - strFree(&sql); - strPrintf(&ins, "INSERT INTO %s", zId); + pStmt = db_prepare("%s", sqlite3_str_value(pSql)); + strFree(pSql); + sqlite3_str_appendf(pIns, "INSERT INTO %s", zId); zSep = "("; for(i=0; az[i]; i++){ - strPrintf(&ins, "%s%s", zSep, az[i]); + sqlite3_str_appendf(pIns, "%s%s", zSep, az[i]); zSep = ","; } - strPrintf(&ins,") VALUES"); + sqlite3_str_appendf(pIns,") VALUES"); namelistFree(az); } nCol = sqlite3_column_count(pStmt); while( SQLITE_ROW==sqlite3_step(pStmt) ){ - fprintf(out, "%s",ins.z); + Wfprintf(out, "%s",sqlite3_str_value(pIns)); zSep = "("; for(i=0; inPk2 ){ zSep = "SELECT "; for(i=0; i1)?", ":""), i); + sqlite3_str_appendf(pSql, "\nORDER BY "); + for(i=1; i<=nPK; i++) sqlite3_str_appendf(pSql, "%s%d", ((i>1)?", ":""), i); } static void rbudiff_one_table(const char *zTab, FILE *out){ @@ -1264,14 +1236,17 @@ static void rbudiff_one_table(const char *zTab, FILE *out){ char **azCol; /* NULL terminated array of col names */ int i; int nCol; - Str ct = {0, 0, 0}; /* The "CREATE TABLE data_xxx" statement */ - Str sql = {0, 0, 0}; /* Query to find differences */ - Str insert = {0, 0, 0}; /* First part of output INSERT statement */ + sqlite3_str *pCt; /* The "CREATE TABLE data_xxx" statement */ + sqlite3_str *pSql; /* Query to find differences */ + sqlite3_str *pInsert; /* First part of output INSERT statement */ sqlite3_stmt *pStmt = 0; int nRow = 0; /* Total rows in data_xxx table */ /* --rbu mode must use real primary keys. */ g.bSchemaPK = 1; + pCt = sqlite3_str_new(0); + pSql = sqlite3_str_new(0); + pInsert = sqlite3_str_new(0); /* Check that the schemas of the two tables match. Exit early otherwise. */ checkSchemasMatch(zTab); @@ -1285,35 +1260,35 @@ static void rbudiff_one_table(const char *zTab, FILE *out){ for(nCol=0; azCol[nCol]; nCol++); /* Build and output the CREATE TABLE statement for the data_xxx table */ - strPrintf(&ct, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab); - if( bOtaRowid ) strPrintf(&ct, "rbu_rowid, "); - strPrintfArray(&ct, ", ", "%s", &azCol[bOtaRowid], -1); - strPrintf(&ct, ", rbu_control);"); + sqlite3_str_appendf(pCt, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab); + if( bOtaRowid ) sqlite3_str_appendf(pCt, "rbu_rowid, "); + strPrintfArray(pCt, ", ", "%s", &azCol[bOtaRowid], -1); + sqlite3_str_appendf(pCt, ", rbu_control);"); /* Get the SQL for the query to retrieve data from the two databases */ - getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, &sql); + getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, pSql); /* Build the first part of the INSERT statement output for each row ** in the data_xxx table. */ - strPrintf(&insert, "INSERT INTO 'data_%q' (", zTab); - if( bOtaRowid ) strPrintf(&insert, "rbu_rowid, "); - strPrintfArray(&insert, ", ", "%s", &azCol[bOtaRowid], -1); - strPrintf(&insert, ", rbu_control) VALUES("); + sqlite3_str_appendf(pInsert, "INSERT INTO 'data_%q' (", zTab); + if( bOtaRowid ) sqlite3_str_appendf(pInsert, "rbu_rowid, "); + strPrintfArray(pInsert, ", ", "%s", &azCol[bOtaRowid], -1); + sqlite3_str_appendf(pInsert, ", rbu_control) VALUES("); - pStmt = db_prepare("%s", sql.z); + pStmt = db_prepare("%s", sqlite3_str_value(pSql)); while( sqlite3_step(pStmt)==SQLITE_ROW ){ /* If this is the first row output, print out the CREATE TABLE - ** statement first. And then set ct.z to NULL so that it is not + ** statement first. And reset pCt so that it will not be ** printed again. */ - if( ct.z ){ - fprintf(out, "%s\n", ct.z); - strFree(&ct); + if( sqlite3_str_length(pCt) ){ + fprintf(out, "%s\n", sqlite3_str_value(pCt)); + sqlite3_str_reset(pCt); } /* Output the first part of the INSERT statement */ - fprintf(out, "%s", insert.z); + fprintf(out, "%s", sqlite3_str_value(pInsert)); nRow++; if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){ @@ -1369,15 +1344,16 @@ static void rbudiff_one_table(const char *zTab, FILE *out){ sqlite3_finalize(pStmt); if( nRow>0 ){ - Str cnt = {0, 0, 0}; - strPrintf(&cnt, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow); - fprintf(out, "%s\n", cnt.z); - strFree(&cnt); + sqlite3_str *pCnt = sqlite3_str_new(0); + sqlite3_str_appendf(pCnt, + "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow); + fprintf(out, "%s\n", sqlite3_str_value(pCnt)); + strFree(pCnt); } - strFree(&ct); - strFree(&sql); - strFree(&insert); + strFree(pCt); + strFree(pSql); + strFree(pInsert); } /* @@ -1399,25 +1375,25 @@ static void summarize_one_table(const char *zTab, FILE *out){ int n2; /* Number of columns in aux */ int i; /* Loop counter */ const char *zSep; /* Separator string */ - Str sql; /* Comparison query */ + sqlite3_str *pSql; /* Comparison query */ sqlite3_stmt *pStmt; /* Query statement to do the diff */ sqlite3_int64 nUpdate; /* Number of updated rows */ sqlite3_int64 nUnchanged; /* Number of unmodified rows */ sqlite3_int64 nDelete; /* Number of deleted rows */ sqlite3_int64 nInsert; /* Number of inserted rows */ - strInit(&sql); + pSql = sqlite3_str_new(0); if( sqlite3_table_column_metadata(g.db,"aux",zTab,0,0,0,0,0,0) ){ if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){ /* Table missing from second database. */ - fprintf(out, "%s: missing from second database\n", zTab); + Wfprintf(out, "%s: missing from second database\n", zTab); } goto end_summarize_one_table; } if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){ /* Table missing from source */ - fprintf(out, "%s: missing from first database\n", zTab); + Wfprintf(out, "%s: missing from first database\n", zTab); goto end_summarize_one_table; } @@ -1434,57 +1410,57 @@ static void summarize_one_table(const char *zTab, FILE *out){ || az[n] ){ /* Schema mismatch */ - fprintf(out, "%s: incompatible schema\n", zTab); + Wfprintf(out, "%s: incompatible schema\n", zTab); goto end_summarize_one_table; } /* Build the comparison query */ for(n2=n; az[n2]; n2++){} - strPrintf(&sql, "SELECT 1, count(*)"); + sqlite3_str_appendf(pSql, "SELECT 1, count(*)"); if( n2==nPk2 ){ - strPrintf(&sql, ", 0\n"); + sqlite3_str_appendf(pSql, ", 0\n"); }else{ zSep = ", sum("; for(i=nPk; az[i]; i++){ - strPrintf(&sql, "%sA.%s IS NOT B.%s", zSep, az[i], az[i]); + sqlite3_str_appendf(pSql, "%sA.%s IS NOT B.%s", zSep, az[i], az[i]); zSep = " OR "; } - strPrintf(&sql, ")\n"); + sqlite3_str_appendf(pSql, ")\n"); } - strPrintf(&sql, " FROM main.%s A, aux.%s B\n", zId, zId); + sqlite3_str_appendf(pSql, " FROM main.%s A, aux.%s B\n", zId, zId); zSep = " WHERE"; for(i=0; inPk ){ - strPrintf(&sql, "SELECT %d", SQLITE_UPDATE); + sqlite3_str_appendf(pSql, "SELECT %d", SQLITE_UPDATE); for(i=0; i Date: Wed, 3 Jan 2024 14:13:27 +0000 Subject: [PATCH 20/24] Enhance sqlite3_analyzer.exe so that it uses the ext/consio extension. FossilOrigin-Name: 769de0b98e136e4a0945b80216d0c9583c1ccd9de69cb0494875c2300e172646 --- Makefile.msc | 2 +- ext/consio/console_io.c | 4 +++- ext/consio/console_io.h | 2 +- manifest | 18 +++++++-------- manifest.uuid | 2 +- tool/sqlite3_analyzer.c.in | 47 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 62 insertions(+), 13 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index d862fbbfb1..19bfe2f386 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2550,7 +2550,7 @@ smoketest: $(TESTPROGS) shelltest: $(TESTPROGS) .\testfixture.exe $(TOP)\test\permutations.test shell -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP) +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\consio\console_io.h $(TOP)\ext\consio\console_io.c $(SQLITE_TCL_DEP) $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) diff --git a/ext/consio/console_io.c b/ext/consio/console_io.c index da62321da8..3acb0daa27 100755 --- a/ext/consio/console_io.c +++ b/ext/consio/console_io.c @@ -24,9 +24,11 @@ # include # include # include -# include "console_io.h" # include "sqlite3.h" #endif +#ifndef HAVE_CONSOLE_IO_H +# include "console_io.h" +#endif #ifndef SQLITE_CIO_NO_TRANSLATE # if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT diff --git a/ext/consio/console_io.h b/ext/consio/console_io.h index 98a87db3db..26fd7dd946 100644 --- a/ext/consio/console_io.h +++ b/ext/consio/console_io.h @@ -28,7 +28,7 @@ ** CIO_WIN_WC_XLATE is defined as 0 or 1, reflecting whether console I/O ** translation for Windows is effected for the build. */ - +#define HAVE_CONSOLE_IO_H 1 #ifndef SQLITE_INTERNAL_LINKAGE # define SQLITE_INTERNAL_LINKAGE extern /* external to translation unit */ # include diff --git a/manifest b/manifest index 7aa5d0ca33..7e01175314 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Update\sthe\ssqldiff.exe\sutility\sprogram\sso\sthat\sit\suses\sthe\ssqlite3_str\nstring\sinterface,\sand\sso\sthat\sit\sdoes\sconsole\soutput\susing\sthe\next/consio\sextension. -D 2024-01-03T12:26:02.221 +C Enhance\ssqlite3_analyzer.exe\sso\sthat\sit\suses\sthe\sext/consio\sextension. +D 2024-01-03T14:13:27.590 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 24be65ae641c5727bbc247d60286a15ecc24fb80f14f8fb2d32533bf0ec96e79 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 8d4860af68385187806e16757bcbca0b852a7ae1f0618fa2eb42fe51166c5bb6 +F Makefile.msc df56c06ef05add5ebcdcc9d4823fb2ec66ac16b06acb6971a7420859025886fa F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3 F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -51,8 +51,8 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94 F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a -F ext/consio/console_io.c 097323f60037d70523421f0248958d2280851f8ff2f9a443d4ee1474c4769118 x -F ext/consio/console_io.h 3228dff1717481202a24f6dcf45ce0b75e7a778bf6877089518a44e1473b76a3 +F ext/consio/console_io.c e1be639e79e54264b3ae97ca291728987a9aa82e6a4526458e6400f5e083e524 x +F ext/consio/console_io.h 0548b83d7c4b7270ad544a67f2bb90cebc519637fa39b1838df4744cf0d87646 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 F ext/expert/expert1.test 0dd5cb096d66bed593e33053a3b364f6ef52ed72064bf5cf298364636dbf3cd6 @@ -2118,7 +2118,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 985452ffc8554baee0f945d078859d393a22abc0bbf553a9f12acae1b6e1fdd0 -F tool/sqlite3_analyzer.c.in f88615bf33098945e0a42f17733f472083d150b58bdaaa5555a7129d0a51621c +F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c 41c586dee84d0b190ad13e0282ed83d4a65ec9fefde9adf4943efdf6558eea7f @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 704943e96f2620b99260667ac9922c2f72bc3e92e2dfe1d9c2a91c7b704564d9 -R e56f90f682975bdbaf88cd4673a534d1 +P 4443b7e592da97d1cb1b3b79ed0559452d8057a33aba4d184c2fffbf200e05f5 +R f76eccd563bb40276e5cf372a3372f6a U drh -Z a3427ef23d574bfd370854aa9641acf2 +Z 389ec51294279a741a10f4bf32e2923d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 00c224a725..d810db232a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4443b7e592da97d1cb1b3b79ed0559452d8057a33aba4d184c2fffbf200e05f5 \ No newline at end of file +769de0b98e136e4a0945b80216d0c9583c1ccd9de69cb0494875c2300e172646 \ No newline at end of file diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in index d8b000209f..2d799ed250 100644 --- a/tool/sqlite3_analyzer.c.in +++ b/tool/sqlite3_analyzer.c.in @@ -19,8 +19,55 @@ INCLUDE sqlite3.c #endif INCLUDE $ROOT/src/tclsqlite.c +#if defined(_WIN32) +INCLUDE $ROOT/ext/consio/console_io.h +INCLUDE $ROOT/ext/consio/console_io.c + +/* Substitute "puts" command. Only these forms recognized: +** +** puts STRING +** puts stderr STRING +** puts -nonewline STRING +*/ +static int subst_puts( + void *NotUsed, + Tcl_Interp *interp, + int objc, + Tcl_Obj *const*objv +){ + FILE *pOut = stdout; + const char *zOut; + int addNewLine = 1; + if( objc==2 ){ + zOut = Tcl_GetString(objv[1]); + }else if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?stderr|-nonewline? STRING"); + return TCL_ERROR; + }else{ + const char *zArg = Tcl_GetString(objv[1]); + if( zArg==0 ) return TCL_ERROR; + zOut = Tcl_GetString(objv[2]); + if( strcmp(zArg, "stderr")==0 ){ + pOut = stderr; + }else if( strcmp(zArg, "-nonewline")==0 ){ + addNewLine = 0; + }else{ + Tcl_AppendResult(interp, "bad argument: ", zArg, 0); + return TCL_ERROR; + } + } + fPutsUtf8(zOut, pOut); + if( addNewLine ) fPutsUtf8("\n", pOut); + return TCL_OK; +} +#endif /* defined(_WIN32) */ + const char *sqlite3_analyzer_init_proc(Tcl_Interp *interp){ +#if defined(_WIN32) + Tcl_CreateObjCommand(interp, "puts", subst_puts, 0, 0); +#else (void)interp; +#endif return BEGIN_STRING INCLUDE $ROOT/tool/spaceanal.tcl From 433e904d99c06bbd818bd772d3dd4f6e5e049291 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 3 Jan 2024 15:49:04 +0000 Subject: [PATCH 21/24] Change a constant from decimal to hex to avoid a compiler warning on Mac. FossilOrigin-Name: e3acb8a43ad544fd5b5341058276bd3b61b6bdb6b719790476a90e0de4320f90 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteLimit.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7e01175314..1371a0e2ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\ssqlite3_analyzer.exe\sso\sthat\sit\suses\sthe\sext/consio\sextension. -D 2024-01-03T14:13:27.590 +C Change\sa\sconstant\sfrom\sdecimal\sto\shex\sto\savoid\sa\scompiler\swarning\son\sMac. +D 2024-01-03T15:49:04.315 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -743,7 +743,7 @@ F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 F src/sqliteInt.h 73800d73e21180e6b3df8d0fe7d11758dc24367fd2b0b0075b48fc116de406bb -F src/sqliteLimit.h 0c4b40bfebb193b5a4eb13db928413e618be8b05fcb572ddf03664c036568f7f +F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c ecbc3c99c0d0c3ed122a913f143026c26d38d57f33e06bb71185dd5c1efe37cd @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4443b7e592da97d1cb1b3b79ed0559452d8057a33aba4d184c2fffbf200e05f5 -R f76eccd563bb40276e5cf372a3372f6a +P 769de0b98e136e4a0945b80216d0c9583c1ccd9de69cb0494875c2300e172646 +R e916baf59c351e9547549cf9c20af1c2 U drh -Z 389ec51294279a741a10f4bf32e2923d +Z 9cefdf7e48b570cbd02d35dc60bf134c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d810db232a..fc82a7ab34 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -769de0b98e136e4a0945b80216d0c9583c1ccd9de69cb0494875c2300e172646 \ No newline at end of file +e3acb8a43ad544fd5b5341058276bd3b61b6bdb6b719790476a90e0de4320f90 \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 0eaadec1cb..abf59e1b3a 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -187,7 +187,7 @@ ** max_page_count macro. */ #ifndef SQLITE_MAX_PAGE_COUNT -# define SQLITE_MAX_PAGE_COUNT 4294967294 +# define SQLITE_MAX_PAGE_COUNT 0xfffffffe /* 4294967294 */ #endif /* From 4093b296105503255c5559f8a6b55b7042844dc9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 3 Jan 2024 16:41:50 +0000 Subject: [PATCH 22/24] Convert the JSON functions to use lookaside memory allocation whenever feasible, to avoid hitting the global memory allocator mutex. FossilOrigin-Name: a79a244954f728596da3c0e28fa3b887258d1bd831f53881970f418f3fba84c7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 44 +++++++++++++++++++++++++++----------------- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 1371a0e2ad..20344e2b71 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sa\sconstant\sfrom\sdecimal\sto\shex\sto\savoid\sa\scompiler\swarning\son\sMac. -D 2024-01-03T15:49:04.315 +C Convert\sthe\sJSON\sfunctions\sto\suse\slookaside\smemory\sallocation\swhenever\nfeasible,\sto\savoid\shitting\sthe\sglobal\smemory\sallocator\smutex. +D 2024-01-03T16:41:50.573 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -697,7 +697,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a84de35816b8aebf6a97e08887cce486c0591d0b6147088fa888325f0c17f9ef +F src/json.c 4913fd22c4f0fa30643afb93a4d78d289cd490620e782b31016c3d4b2049b1cc F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 769de0b98e136e4a0945b80216d0c9583c1ccd9de69cb0494875c2300e172646 -R e916baf59c351e9547549cf9c20af1c2 +P e3acb8a43ad544fd5b5341058276bd3b61b6bdb6b719790476a90e0de4320f90 +R 951d215a027ca002df153a3678fabcb5 U drh -Z 9cefdf7e48b570cbd02d35dc60bf134c +Z 4bd172c52ecc91bae1ff8ea6834cc34c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fc82a7ab34..dac2656e75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3acb8a43ad544fd5b5341058276bd3b61b6bdb6b719790476a90e0de4320f90 \ No newline at end of file +a79a244954f728596da3c0e28fa3b887258d1bd831f53881970f418f3fba84c7 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6ee51d39ca..682f30597f 100644 --- a/src/json.c +++ b/src/json.c @@ -298,6 +298,7 @@ struct JsonParse { u32 nBlob; /* Bytes of aBlob[] actually used */ u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ char *zJson; /* Json text used for parsing */ + sqlite3 *db; /* The database connection to which this object belongs */ int nJson; /* Length of the zJson string in bytes */ u32 nJPRef; /* Number of references to this object */ u32 iErr; /* Error location in zJson[] */ @@ -823,7 +824,7 @@ static void jsonParseReset(JsonParse *pParse){ pParse->bJsonIsRCStr = 0; } if( pParse->nBlobAlloc ){ - sqlite3_free(pParse->aBlob); + sqlite3DbFree(pParse->db, pParse->aBlob); pParse->aBlob = 0; pParse->nBlob = 0; pParse->nBlobAlloc = 0; @@ -840,7 +841,7 @@ static void jsonParseFree(JsonParse *pParse){ pParse->nJPRef--; }else{ jsonParseReset(pParse); - sqlite3_free(pParse); + sqlite3DbFree(pParse->db, pParse); } } } @@ -1071,7 +1072,7 @@ static int jsonBlobExpand(JsonParse *pParse, u32 N){ t = pParse->nBlobAlloc*2; } if( taBlob, t ); + aNew = sqlite3DbRealloc(pParse->db, pParse->aBlob, t); if( aNew==0 ){ pParse->oom = 1; return 1; } pParse->aBlob = aNew; pParse->nBlobAlloc = t; @@ -1996,14 +1997,15 @@ static void jsonReturnStringAsBlob(JsonString *pStr){ jsonStringTerminate(pStr); px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; + px.db = sqlite3_context_db_handle(pStr->pCtx); (void)jsonTranslateTextToBlob(&px, 0); if( px.oom ){ - sqlite3_free(px.aBlob); + sqlite3DbFree(px.db, px.aBlob); sqlite3_result_error_nomem(pStr->pCtx); }else{ assert( px.nBlobAlloc>0 ); assert( !px.bReadOnly ); - sqlite3_result_blob(pStr->pCtx, px.aBlob, px.nBlob, sqlite3_free); + sqlite3_result_blob(pStr->pCtx, px.aBlob, px.nBlob, SQLITE_DYNAMIC); } } @@ -2619,6 +2621,7 @@ static u32 jsonCreateEditSubstructure( static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; int rc; memset(pIns, 0, sizeof(*pIns)); + pIns->db = pParse->db; if( zTail[0]==0 ){ /* No substructure. Just insert what is given in pParse. */ pIns->aBlob = pParse->aIns; @@ -2746,6 +2749,7 @@ static u32 jsonLookupStep( testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); memset(&ix, 0, sizeof(ix)); + ix.db = pParse->db; jsonBlobAppendNode(&ix, rawKey?JSONB_TEXTRAW:JSONB_TEXT5, nKey, 0); pParse->oom |= ix.oom; rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i]); @@ -2956,7 +2960,7 @@ static void jsonReturnFromBlob( char *zOut; u32 nOut = sz; z = (const char*)&pParse->aBlob[i+n]; - zOut = sqlite3_malloc( nOut+1 ); + zOut = sqlite3DbMallocRaw(db, nOut+1); if( zOut==0 ) goto returnfromblob_oom; for(iIn=iOut=0; iIndb = sqlite3_context_db_handle(ctx); switch( eType ){ default: { pParse->aBlob = aNull; @@ -3068,7 +3073,7 @@ static int jsonFunctionArgToBlob( pParse->nJson = nJson; if( jsonConvertTextToBlob(pParse, ctx) ){ sqlite3_result_error(ctx, "malformed JSON", -1); - sqlite3_free(pParse->aBlob); + sqlite3DbFree(pParse->db, pParse->aBlob); memset(pParse, 0, sizeof(pParse[0])); return 1; } @@ -3225,6 +3230,7 @@ static JsonParse *jsonParseFuncArg( int eType; /* Datatype of pArg */ JsonParse *p = 0; /* Value to be returned */ JsonParse *pFromCache = 0; /* Value taken from cache */ + sqlite3 *db; /* The database connection */ assert( ctx!=0 ); eType = sqlite3_value_type(pArg); @@ -3238,14 +3244,16 @@ static JsonParse *jsonParseFuncArg( return pFromCache; } } + db = sqlite3_context_db_handle(ctx); rebuild_from_cache: - p = sqlite3_malloc64( sizeof(*p) ); + p = sqlite3DbMallocZero(db, sizeof(*p)); if( p==0 ) goto json_pfa_oom; memset(p, 0, sizeof(*p)); + p->db = db; p->nJPRef = 1; if( pFromCache!=0 ){ u32 nBlob = pFromCache->nBlob; - p->aBlob = sqlite3_malloc64( nBlob ); + p->aBlob = sqlite3DbMallocRaw(db, nBlob); if( p->aBlob==0 ) goto json_pfa_oom; memcpy(p->aBlob, pFromCache->aBlob, nBlob); p->nBlobAlloc = p->nBlob = nBlob; @@ -3501,7 +3509,7 @@ static void jsonParseFunc( if( p==0 ) return; if( argc==1 ){ jsonDebugPrintBlob(p, 0, p->nBlob, 0, &out); - sqlite3_result_text64(ctx, out.zText, out.nChar, sqlite3_free, SQLITE_UTF8); + sqlite3_result_text64(ctx, out.zText, out.nChar, SQLITE_DYNAMIC, SQLITE_UTF8); }else{ jsonShowParse(p); } @@ -4320,6 +4328,7 @@ static void jsonErrorFunc( assert( argc==1 ); UNUSED_PARAMETER(argc); memset(&s, 0, sizeof(s)); + s.db = sqlite3_context_db_handle(ctx); if( jsonFuncArgMightBeBinary(argv[0]) ){ s.aBlob = (u8*)sqlite3_value_blob(argv[0]); s.nBlob = sqlite3_value_bytes(argv[0]); @@ -4609,9 +4618,9 @@ static int jsonEachConnect( "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," "json HIDDEN,root HIDDEN)"); if( rc==SQLITE_OK ){ - pNew = (JsonEachConnection*)(*ppVtab = sqlite3_malloc( sizeof(*pNew) )); + pNew = (JsonEachConnection*)sqlite3DbMallocZero(db, sizeof(*pNew)); + *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; - memset(pNew, 0, sizeof(*pNew)); sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); pNew->db = db; } @@ -4620,7 +4629,8 @@ static int jsonEachConnect( /* destructor for json_each virtual table */ static int jsonEachDisconnect(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); + JsonEachConnection *p = (JsonEachConnection*)pVtab; + sqlite3DbFree(p->db, pVtab); return SQLITE_OK; } @@ -4630,9 +4640,8 @@ static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ JsonEachCursor *pCur; UNUSED_PARAMETER(p); - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3DbMallocZero(pVtab->db, sizeof(*pCur)); if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); pCur->db = pVtab->db; jsonStringZero(&pCur->path); *ppCursor = &pCur->base; @@ -4669,7 +4678,7 @@ static int jsonEachClose(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; jsonEachCursorReset(p); - sqlite3_free(cur); + sqlite3DbFree(p->db, cur); return SQLITE_OK; } @@ -5003,6 +5012,7 @@ static int jsonEachFilter( if( idxNum==0 ) return SQLITE_OK; memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; + p->sParse.db = p->db; if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ if( jsonFuncArgMightBeBinary(argv[0]) ){ p->sParse.nBlob = sqlite3_value_bytes(argv[0]); From aa08f885ba7990d53a78cc321ad1b71d868a2c3a Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 3 Jan 2024 20:40:17 +0000 Subject: [PATCH 23/24] Fix a #ifdef in sqlite3_test_control() that was preventing builds with SQLITE_OMIT_WSD. FossilOrigin-Name: d546a9c94caf7408cc6e4530ec190d3a13fae09dc15b71b03d6369e02ee62abd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 20344e2b71..c62b32c739 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sthe\sJSON\sfunctions\sto\suse\slookaside\smemory\sallocation\swhenever\nfeasible,\sto\savoid\shitting\sthe\sglobal\smemory\sallocator\smutex. -D 2024-01-03T16:41:50.573 +C Fix\sa\s#ifdef\sin\ssqlite3_test_control()\sthat\swas\spreventing\sbuilds\swith\nSQLITE_OMIT_WSD. +D 2024-01-03T20:40:17.581 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -700,7 +700,7 @@ F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 F src/json.c 4913fd22c4f0fa30643afb93a4d78d289cd490620e782b31016c3d4b2049b1cc F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 -F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 +F src/main.c 438b95162acfa17b7d218f586f5bde11d6ae82bcf030c9611fc537556870ad6b F src/malloc.c f016922435dc7d1f1f5083a03338a3e91f8c67ce2c5bdcfa4cdef62e612f5fcc F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e3acb8a43ad544fd5b5341058276bd3b61b6bdb6b719790476a90e0de4320f90 -R 951d215a027ca002df153a3678fabcb5 +P a79a244954f728596da3c0e28fa3b887258d1bd831f53881970f418f3fba84c7 +R 2cb03286d1de4c61e0953028252fab0a U drh -Z 4bd172c52ecc91bae1ff8ea6834cc34c +Z a9b64d69ed9b2889c84934f6dec5097d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dac2656e75..5e6843d79e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a79a244954f728596da3c0e28fa3b887258d1bd831f53881970f418f3fba84c7 \ No newline at end of file +d546a9c94caf7408cc6e4530ec190d3a13fae09dc15b71b03d6369e02ee62abd \ No newline at end of file diff --git a/src/main.c b/src/main.c index 50572ef87f..03429983d6 100644 --- a/src/main.c +++ b/src/main.c @@ -4671,7 +4671,7 @@ int sqlite3_test_control(int op, ...){ ** to be the current setting. */ case SQLITE_TESTCTRL_JSON_SELFCHECK: { -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) int *pOnOff = va_arg(ap, int*); if( *pOnOff<0 ){ *pOnOff = sqlite3Config.bJsonSelfcheck; From 267721e72aeb1a3258acf77ada5a4501c0d1ac6c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 4 Jan 2024 13:01:02 +0000 Subject: [PATCH 24/24] Restructure some code to fix what appears to be a false-positive UBSAN warning. FossilOrigin-Name: fe952c12903ea2150880c8bb57cda2efc00ce9fa801568a68c619e0745f30567 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/build.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c62b32c739..0689155ca9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\s#ifdef\sin\ssqlite3_test_control()\sthat\swas\spreventing\sbuilds\swith\nSQLITE_OMIT_WSD. -D 2024-01-03T20:40:17.581 +C Restructure\ssome\scode\sto\sfix\swhat\sappears\sto\sbe\sa\sfalse-positive\sUBSAN\swarning. +D 2024-01-04T13:01:02.910 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -679,7 +679,7 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c dee25e097b749275333b55d64a5ffc079249576f8e88a2ee476468cf67510f4b F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240 F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062 -F src/build.c 386eadecabe2e99a3783eb802ca01e665f8e0c2af0e0aab28161fd7def219a9d +F src/build.c f640dbe723e35f29a91870ff8bd9fb015b1cbc49666014b1d422c012eb333de9 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 23331529e654be40ca97d171cbbffe9b3d4c71cc53b78fe5501230675952da8b @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a79a244954f728596da3c0e28fa3b887258d1bd831f53881970f418f3fba84c7 -R 2cb03286d1de4c61e0953028252fab0a +P d546a9c94caf7408cc6e4530ec190d3a13fae09dc15b71b03d6369e02ee62abd +R e1f7197b52fbcc05b1cd044084aa5ecc U drh -Z a9b64d69ed9b2889c84934f6dec5097d +Z 2859a9df9b5c4d7ab26ab170564bfb9b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5e6843d79e..46b4fd2c98 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d546a9c94caf7408cc6e4530ec190d3a13fae09dc15b71b03d6369e02ee62abd \ No newline at end of file +fe952c12903ea2150880c8bb57cda2efc00ce9fa801568a68c619e0745f30567 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 50ce97b51a..d2e9e0bc03 100644 --- a/src/build.c +++ b/src/build.c @@ -1655,7 +1655,8 @@ char sqlite3AffinityType(const char *zIn, Column *pCol){ assert( zIn!=0 ); while( zIn[0] ){ - h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; + u8 x = *(u8*)zIn; + h = (h<<8) + sqlite3UpperToLower[x]; zIn++; if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ aff = SQLITE_AFF_TEXT;