From cbe4a26e028e1b0607b4db78b134812a28c710ec Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 21 Apr 2025 19:53:12 +0000 Subject: [PATCH 01/42] Do a better job of providing bug compatibility with SQLite 3.44.0. See [forum:/forumpost/07e206fcd6|forum thread 07e206fcd6] for background. FossilOrigin-Name: 614d061b32c3bdf4825323d6f8e8a98624e4eeeb96dd442d8e365527122ed3bc --- manifest | 14 +++---- manifest.uuid | 2 +- src/json.c | 109 ++++++++++++++++++++------------------------------ 3 files changed, 51 insertions(+), 74 deletions(-) diff --git a/manifest b/manifest index ddbb92a052..6f71143f32 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sto\scomments\sin\ssqlite3session.h. -D 2025-04-19T20:08:04.182 +C Do\sa\sbetter\sjob\sof\sproviding\sbug\scompatibility\swith\sSQLite\s3.44.0.\s\sSee\n[forum:/forumpost/07e206fcd6|forum\sthread\s07e206fcd6]\sfor\sbackground. +D 2025-04-21T19:53:12.406 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -746,7 +746,7 @@ F src/hash.h 46b92795a95bfefb210f52f0c316e9d7cdbcdd7e7fcfb0d8be796d3a5767cddf F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c d05934dfab2c5c0c480fc6fd2038f11215661de08ea6ff38d2563216bd555c1b -F src/json.c d0e1c62a9689018fe5876b1a1eb574e4ab7a40e0928e0e5adda8b3860c70e1b8 +F src/json.c d13d5e29d18b1b711d832cab48f48fab6705747abbac0ab65953c06eba74c99d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 07f78d917ffcdf327982840cfd8e855fd000527a2ea5ace372ce4febcbd0bf97 @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 00313d15270b36fce44f7784c97ddc42517e23c209c1ccc4d814ef06b11528c0 -R 8e876785cd8a9f724c2edd8d86d424db -U dan -Z ad6897ab80a5e3b64556d94fd097e39b +P ea9acb5573f4d71a314e4467d30477a1d01c8db648985750a42b3c047f404c9c +R ead9acfd8b581f4510534b8b584f9293 +U drh +Z dd601f105d2c079fb81118dbb59f331e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 16310575a6..8beea29ca5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea9acb5573f4d71a314e4467d30477a1d01c8db648985750a42b3c047f404c9c +614d061b32c3bdf4825323d6f8e8a98624e4eeeb96dd442d8e365527122ed3bc diff --git a/src/json.c b/src/json.c index 9c38bde985..72b52efd7c 100644 --- a/src/json.c +++ b/src/json.c @@ -398,7 +398,7 @@ struct JsonParse { ** Forward references **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); -static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); +static int jsonArgIsJsonb(sqlite3_value *pJson, JsonParse *p); static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); @@ -816,11 +816,9 @@ static void jsonAppendSqlValue( break; } default: { - if( jsonFuncArgMightBeBinary(pValue) ){ - JsonParse px; - memset(&px, 0, sizeof(px)); - px.aBlob = (u8*)sqlite3_value_blob(pValue); - px.nBlob = sqlite3_value_bytes(pValue); + JsonParse px; + memset(&px, 0, sizeof(px)); + if( jsonArgIsJsonb(pValue, &px) ){ jsonTranslateBlobToText(&px, 0, p); }else if( p->eErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); @@ -1396,7 +1394,7 @@ static u32 jsonbValidityCheck( case JSONB_INT5: { if( sz<3 ) return i+1; j = i+n; - if( z[j]=='-' ){ + if( z[j]=='-' || z[j]=='+' ){ if( sz<4 ) return i+1; j++; } @@ -2494,33 +2492,6 @@ static u32 jsonTranslateBlobToPrettyText( return i; } - -/* Return true if the input pJson -** -** For performance reasons, this routine does not do a detailed check of the -** input BLOB to ensure that it is well-formed. Hence, false positives are -** possible. False negatives should never occur, however. -*/ -static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ - u32 sz, n; - const u8 *aBlob; - int nBlob; - JsonParse s; - if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; - aBlob = sqlite3_value_blob(pJson); - nBlob = sqlite3_value_bytes(pJson); - if( nBlob<1 ) return 0; - if( NEVER(aBlob==0) || (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; - memset(&s, 0, sizeof(s)); - s.aBlob = (u8*)aBlob; - s.nBlob = nBlob; - n = jsonbPayloadSize(&s, 0, &sz); - if( n==0 ) return 0; - if( sz+n!=(u32)nBlob ) return 0; - if( (aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0 ) return 0; - return sz+n==(u32)nBlob; -} - /* ** Given that a JSONB_ARRAY object starts at offset i, return ** the number of entries in that array. @@ -3348,10 +3319,7 @@ static int jsonFunctionArgToBlob( return 0; } case SQLITE_BLOB: { - if( jsonFuncArgMightBeBinary(pArg) ){ - pParse->aBlob = (u8*)sqlite3_value_blob(pArg); - pParse->nBlob = sqlite3_value_bytes(pArg); - }else{ + if( !jsonArgIsJsonb(pArg, pParse) ){ sqlite3_result_error(ctx, "JSON cannot hold BLOB values", -1); return 1; } @@ -3502,27 +3470,43 @@ jsonInsertIntoBlob_patherror: /* ** If pArg is a blob that seems like a JSONB blob, then initialize ** p to point to that JSONB and return TRUE. If pArg does not seem like -** a JSONB blob, then return FALSE; +** a JSONB blob, then return FALSE. ** -** This routine is only called if it is already known that pArg is a -** blob. The only open question is whether or not the blob appears -** to be a JSONB blob. +** For small BLOBs (having no more than 7 bytes of payload) a full +** validity check is done. So for small BLOBs this routine only returns +** true if the value is guaranteed to be a valid JSONB. For larger BLOBs +** (8 byte or more of payload) only the size of the outermost element is +** checked to verify that the BLOB is superficially valid JSONB. +** +** A full JSONB validation is done on smaller BLOBs because those BLOBs might +** also be text JSON that has been incorrectly cast into a BLOB. +** (See tag-20240123-a and https://sqlite.org/forum/forumpost/012136abd5) +** If the BLOB is 9 bytes are larger, then it is not possible for the +** superficial size check done here to pass if the input is really text +** JSON so we do not need to look deeper in that case. +** +** Why we only need to do full JSONB validation for smaller BLOBs: +** +** The first byte of valid JSON text must be one of: '{', '[', '"', ' ', '\n', +** '\r', '\t', '-', or a digit '0' through '9'. Of these, only a subset +** can also be the first byte of JSONB: '{', '[', '\n', '\t', and digits '3' +** through '9'. In every one of those cases, the payload size is 7 bytes +** or less. So if we do full JSONB validation for every BLOB where the +** payload is less than 7 bytes, we will never get a false positive for +** JSONB on an input that is really text JSON. */ static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){ u32 n, sz = 0; + if( sqlite3_value_type(pArg)!=SQLITE_BLOB ) return 0; p->aBlob = (u8*)sqlite3_value_blob(pArg); p->nBlob = (u32)sqlite3_value_bytes(pArg); - if( p->nBlob==0 ){ - p->aBlob = 0; - return 0; - } - if( NEVER(p->aBlob==0) ){ - return 0; - } - if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT + if( p->nBlob>0 + && ALWAYS(p->aBlob!=0) + && (p->aBlob[0] & 0x0f)<=JSONB_OBJECT && (n = jsonbPayloadSize(p, 0, &sz))>0 && sz+n==p->nBlob && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0) + && (sz>7 || (sz&1)==0 || jsonbValidityCheck(p, 0, p->nBlob, 1)==0) ){ return 1; } @@ -4615,21 +4599,17 @@ static void jsonValidFunc( return; } case SQLITE_BLOB: { - if( jsonFuncArgMightBeBinary(argv[0]) ){ - if( flags & 0x04 ){ + JsonParse py; + memset(&py, 0, sizeof(py)); + if( jsonArgIsJsonb(argv[0], &py) ){ + if( (flags & 0x04)!=0 || py.nBlob<=7 ){ /* Superficial checking only - accomplished by the - ** jsonFuncArgMightBeBinary() call above. */ + ** jsonArgIsJsonb() call above. */ res = 1; }else if( flags & 0x08 ){ /* Strict checking. Check by translating BLOB->TEXT->BLOB. If ** no errors occur, call that a "strict check". */ - JsonParse px; - u32 iErr; - memset(&px, 0, sizeof(px)); - px.aBlob = (u8*)sqlite3_value_blob(argv[0]); - px.nBlob = sqlite3_value_bytes(argv[0]); - iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); - res = iErr==0; + res = 0==jsonbValidityCheck(&py, 0, py.nBlob, 1); } break; } @@ -4687,9 +4667,7 @@ static void jsonErrorFunc( UNUSED_PARAMETER(argc); memset(&s, 0, sizeof(s)); s.db = sqlite3_context_db_handle(ctx); - if( jsonFuncArgMightBeBinary(argv[0]) ){ - s.aBlob = (u8*)sqlite3_value_blob(argv[0]); - s.nBlob = sqlite3_value_bytes(argv[0]); + if( jsonArgIsJsonb(argv[0], &s) ){ iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 1); }else{ s.zJson = (char*)sqlite3_value_text(argv[0]); @@ -5374,9 +5352,8 @@ static int jsonEachFilter( memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; p->sParse.db = p->db; - if( jsonFuncArgMightBeBinary(argv[0]) ){ - p->sParse.nBlob = sqlite3_value_bytes(argv[0]); - p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); + if( jsonArgIsJsonb(argv[0], &p->sParse) ){ + /* We have JSONB */ }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); From 81cde80f7b0fec5bf64fb450e94a62047fb5c5fd Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 21 Apr 2025 20:58:49 +0000 Subject: [PATCH 02/42] Further improvements to the decision of whether or not a BLOB input is JSONB. FossilOrigin-Name: 6538813cb89f6109727481e29633e2e98f98e0257c58695e3b53e8ce237d9195 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 13 ++++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 6f71143f32..949c0e4986 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\sa\sbetter\sjob\sof\sproviding\sbug\scompatibility\swith\sSQLite\s3.44.0.\s\sSee\n[forum:/forumpost/07e206fcd6|forum\sthread\s07e206fcd6]\sfor\sbackground. -D 2025-04-21T19:53:12.406 +C Further\simprovements\sto\sthe\sdecision\sof\swhether\sor\snot\sa\sBLOB\sinput\nis\sJSONB. +D 2025-04-21T20:58:49.823 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -746,7 +746,7 @@ F src/hash.h 46b92795a95bfefb210f52f0c316e9d7cdbcdd7e7fcfb0d8be796d3a5767cddf F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c d05934dfab2c5c0c480fc6fd2038f11215661de08ea6ff38d2563216bd555c1b -F src/json.c d13d5e29d18b1b711d832cab48f48fab6705747abbac0ab65953c06eba74c99d +F src/json.c bb0f8f2fa2c0fb4cbcc3d8cc7abdaf161b341d541455069dda310645f031734c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 07f78d917ffcdf327982840cfd8e855fd000527a2ea5ace372ce4febcbd0bf97 @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ea9acb5573f4d71a314e4467d30477a1d01c8db648985750a42b3c047f404c9c -R ead9acfd8b581f4510534b8b584f9293 +P 614d061b32c3bdf4825323d6f8e8a98624e4eeeb96dd442d8e365527122ed3bc +R 8a150a5cec72c6ab77984c7d65063b89 U drh -Z dd601f105d2c079fb81118dbb59f331e +Z 59b3d2a5c6021cf082dd8b7ce8b12041 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8beea29ca5..a056e0af7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -614d061b32c3bdf4825323d6f8e8a98624e4eeeb96dd442d8e365527122ed3bc +6538813cb89f6109727481e29633e2e98f98e0257c58695e3b53e8ce237d9195 diff --git a/src/json.c b/src/json.c index 72b52efd7c..c29caa46bc 100644 --- a/src/json.c +++ b/src/json.c @@ -3489,7 +3489,7 @@ jsonInsertIntoBlob_patherror: ** ** The first byte of valid JSON text must be one of: '{', '[', '"', ' ', '\n', ** '\r', '\t', '-', or a digit '0' through '9'. Of these, only a subset -** can also be the first byte of JSONB: '{', '[', '\n', '\t', and digits '3' +** can also be the first byte of JSONB: '{', '[', and digits '3' ** through '9'. In every one of those cases, the payload size is 7 bytes ** or less. So if we do full JSONB validation for every BLOB where the ** payload is less than 7 bytes, we will never get a false positive for @@ -3497,16 +3497,19 @@ jsonInsertIntoBlob_patherror: */ static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){ u32 n, sz = 0; + u8 c; if( sqlite3_value_type(pArg)!=SQLITE_BLOB ) return 0; p->aBlob = (u8*)sqlite3_value_blob(pArg); p->nBlob = (u32)sqlite3_value_bytes(pArg); if( p->nBlob>0 && ALWAYS(p->aBlob!=0) - && (p->aBlob[0] & 0x0f)<=JSONB_OBJECT + && ((c = p->aBlob[0]) & 0x0f)<=JSONB_OBJECT && (n = jsonbPayloadSize(p, 0, &sz))>0 && sz+n==p->nBlob - && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0) - && (sz>7 || (sz&1)==0 || jsonbValidityCheck(p, 0, p->nBlob, 1)==0) + && ((c & 0x0f)>JSONB_FALSE || sz==0) + && (sz>7 + || (c!=0x7b && c!=0x5b && !sqlite3Isdigit(c)) + || jsonbValidityCheck(p, 0, p->nBlob, 1)==0) ){ return 1; } @@ -4602,7 +4605,7 @@ static void jsonValidFunc( JsonParse py; memset(&py, 0, sizeof(py)); if( jsonArgIsJsonb(argv[0], &py) ){ - if( (flags & 0x04)!=0 || py.nBlob<=7 ){ + if( flags & 0x04 ){ /* Superficial checking only - accomplished by the ** jsonArgIsJsonb() call above. */ res = 1; From d2fd099570172fd82a31c5e2e57800d1fc79b68e Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 21 Apr 2025 23:44:55 +0000 Subject: [PATCH 03/42] Remove an extra conditional that was inserted earlier today and which is unreachable. FossilOrigin-Name: 5ea56af2d27611c5a8ea275316d21a02e70032f93243549ae776e9a89edc6229 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 949c0e4986..1dda006842 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\simprovements\sto\sthe\sdecision\sof\swhether\sor\snot\sa\sBLOB\sinput\nis\sJSONB. -D 2025-04-21T20:58:49.823 +C Remove\san\sextra\sconditional\sthat\swas\sinserted\searlier\stoday\sand\swhich\nis\sunreachable. +D 2025-04-21T23:44:55.916 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -746,7 +746,7 @@ F src/hash.h 46b92795a95bfefb210f52f0c316e9d7cdbcdd7e7fcfb0d8be796d3a5767cddf F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c d05934dfab2c5c0c480fc6fd2038f11215661de08ea6ff38d2563216bd555c1b -F src/json.c bb0f8f2fa2c0fb4cbcc3d8cc7abdaf161b341d541455069dda310645f031734c +F src/json.c c84b0f2bae967341d5a035808d21d0619b23b6e054ceac08e7592cd87f04ed4f F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 07f78d917ffcdf327982840cfd8e855fd000527a2ea5ace372ce4febcbd0bf97 @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 614d061b32c3bdf4825323d6f8e8a98624e4eeeb96dd442d8e365527122ed3bc -R 8a150a5cec72c6ab77984c7d65063b89 +P 6538813cb89f6109727481e29633e2e98f98e0257c58695e3b53e8ce237d9195 +R f846b0d70084164be308ce186bb3a97a U drh -Z 59b3d2a5c6021cf082dd8b7ce8b12041 +Z 29505047856922d4c8d25e24a8f84ffb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a056e0af7c..f8c99133e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6538813cb89f6109727481e29633e2e98f98e0257c58695e3b53e8ce237d9195 +5ea56af2d27611c5a8ea275316d21a02e70032f93243549ae776e9a89edc6229 diff --git a/src/json.c b/src/json.c index c29caa46bc..1528fdb708 100644 --- a/src/json.c +++ b/src/json.c @@ -1394,7 +1394,7 @@ static u32 jsonbValidityCheck( case JSONB_INT5: { if( sz<3 ) return i+1; j = i+n; - if( z[j]=='-' || z[j]=='+' ){ + if( z[j]=='-' ){ if( sz<4 ) return i+1; j++; } From 204d787644733e528b039c60527a3559a3532c96 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 22 Apr 2025 01:01:19 +0000 Subject: [PATCH 04/42] Generic cleanups and fixes in the teaish build. FossilOrigin-Name: 734823a5a8e073b511ebc3f37df3542c1bb734b9b1e1bd50ca3e9df12c4bb89e --- autoconf/tea/Makefile.in | 4 +- autoconf/tea/autosetup/core.tcl | 251 ++++++++++++++++++++---------- autoconf/tea/autosetup/tester.tcl | 2 + autoconf/tea/pkgIndex.tcl.in | 6 +- autoconf/tea/teaish.tcl | 6 +- autoconf/tea/teaish.tester.tcl.in | 6 +- autosetup/proj.tcl | 44 +++--- manifest | 26 ++-- manifest.uuid | 2 +- 9 files changed, 215 insertions(+), 132 deletions(-) diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index 911717bc43..172e381951 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -320,8 +320,8 @@ dist.zip.dir = $(tx.name)-$(tx.version) dist.zip = $(dist.zip.dir).zip .PHONY: dist.zip dist.zip-core dist.zip-post #dist.zip-pre: -# We apparently can't add a pre-hook here, even if dist.zip-pre is -# .PHONY, else "make dist" rebuilds the archive each time it's run. +# We apparently can't add a pre-hook here, else "make dist" rebuilds +# the archive each time it's run. $(dist.zip): $(tx.dist.files) @rm -fr $(teaish__dist.tmp.zip) @mkdir -p $(teaish__dist.tmp.zip)/$(dist.zip.dir) diff --git a/autoconf/tea/autosetup/core.tcl b/autoconf/tea/autosetup/core.tcl index 4c9aee4d66..14044e607a 100644 --- a/autoconf/tea/autosetup/core.tcl +++ b/autoconf/tea/autosetup/core.tcl @@ -32,12 +32,14 @@ array set teaish__Config [proj-strip-hash-comments { debug-enabled 0 # # 0 = don't yet have extension's pkgindex - # 0x01 = teaish__find_extension found TEAISH_DIR/pkgIndex.tcl - # 0x02 = teaish__find_extension found srcdir/pkgIndex.tcl.in - # 0x04 = teaish__find_extension found TEAISH_DIR/pkgIndex.tcl (static file) - # 0x10 = teaish-pragma was called: behave as if 0x04 + # 0x01 = found TEAISH_DIR/pkgIndex.tcl.in + # 0x02 = found srcdir/pkgIndex.tcl.in + # 0x10 = found TEAISH_DIR/pkgIndex.tcl (static file) + # 0x20 = static-pkgIndex.tcl pragma: behave as if 0x10 + # + # Reminder: it's significant that the bottom 4 bits be + # cases where teaish manages ./pkgIndex.tcl. # - # This might no longer be needed. pkgindex-policy 0 # @@ -54,8 +56,9 @@ array set teaish__Config [proj-strip-hash-comments { -libDir TEAISH_LIBDIR_NAME -loadPrefix TEAISH_LOAD_PREFIX -version TEAISH_VERSION - -vsatisfies TEAISH_VSATISFIES_TCL + -vsatisfies TEAISH_VSATISFIES -options {} + -pragmas {} } # @@ -64,9 +67,19 @@ array set teaish__Config [proj-strip-hash-comments { queued-checks-pre {} queued-checks-post {} + # Whether or not "make dist" parts are enabled. They get enabled + # when building from an extension's dir, disabled when building + # elsewhere. + dist-enabled 1 }] set teaish__Config(core-dir) $::autosetup(libdir)/teaish +# +# Array of info managed by teaish-pkginfo-get and friends. Has the +# same set of keys as $teaish__Config(pkginfo-f2d). +# +array set teaish__PkgInfo {} + # # Runs {*}$args if $lvl is <= the current verbosity level, else it has # no side effects. @@ -206,13 +219,13 @@ proc teaish-configure-core {} { -version TEAISH_VERSION -v 0.0.0 -pkgName TEAISH_PKGNAME -e {teaish-pkginfo-get -name} -libDir TEAISH_LIBDIR_NAME -e {join [list \ - [teaish-pkginfo-get -pkgName] \ - [teaish-pkginfo-get -version]]} + [teaish-pkginfo-get -pkgName] \ + [teaish-pkginfo-get -version]] ""} -loadPrefix TEAISH_LOAD_PREFIX -e {string totitle [get-define TEAISH_PKGNAME ""]} - -vsatisfies TEAISH_VSATISFIES_TCL -v 8.5- + -vsatisfies TEAISH_VSATISFIES -v {{Tcl 8.5-}} } { set isDefOnly [expr {"-" eq $pflag}] - if {!$isDefOnly && [info exists ::teaish__Config($pflag)]} { + if {!$isDefOnly && [info exists ::teaish__PkgInfo($pflag)]} { continue; } set got [get-define $key ""] @@ -227,7 +240,7 @@ proc teaish-configure-core {} { #puts "***** defining default $pflag $key {$val} isDefOnly=$isDefOnly got=$got" define $key $val if {!$isDefOnly} { - set ::teaish__Config($pflag) $val + set ::teaish__PkgInfo($pflag) $val } } unset key type val @@ -270,7 +283,7 @@ proc teaish-configure-core {} { } proj-assert {1==$gotExt} "Else we cannot have gotten this far" - teaish__configure-phase1 + teaish__configure_phase1 } @@ -288,7 +301,7 @@ proc teaish-debug {msg} { # Runs "phase 1" of the configuration, immediately after processing # --flags. This is what will import the client-defined teaish.tcl. # -proc teaish__configure-phase1 {} { +proc teaish__configure_phase1 {} { msg-result \ [join [list "Configuring build of Tcl extension" \ [proj-bold [teaish-pkginfo-get -name] \ @@ -365,9 +378,11 @@ proc teaish__configure-phase1 {} { teaish-checks-run -post if {0} { - # Reminder: we cannot do a TEAISH_VSATISFIES_TCL check like the following - # from here because _this_ tcl instance is very possibly not the one - # which will be hosting the extension. + # Reminder: we cannot do a TEAISH_VSATISFIES_TCL check like the + # following from here because _this_ tcl instance is very possibly + # not the one which will be hosting the extension. This part might + # be running in JimTcl, which can't load the being-configured + # extension. if {$::autosetup(istcl)} { # ^^^ this is a canonical Tcl, not JimTcl set vsat [get-define TEAISH_VSATISFIES_TCL ""] @@ -382,7 +397,6 @@ proc teaish__configure-phase1 {} { } } - if {[proj-looks-like-windows]} { # Without this, linking of an extension will not work on Cygwin or # Msys2. @@ -407,6 +421,14 @@ proc teaish__configure-phase1 {} { proj-assert {$tpi ne ""} \ "TEAISH_PKGINDEX_TCL should have been set up by now" teaish__verbose 1 msg-result "Using pkgIndex from $tpi" + if {0x0f & $::teaish__Config(pkgindex-policy)} { + # Don't leave stale pkgIndex.tcl laying around yet don't delete + # or overwrite a user-managed static pkgIndex.tcl. + file delete -force -- [get-define TEAISH_PKGINDEX_TCL] + proj-dot-ins-append [get-define TEAISH_PKGINDEX_TCL_IN] + } else { + teaish-dist-add [file tail $tpi] + } }}; # $::teaish__Config(pkgindex-policy) set dEx $::teaish__Config(teaish-dir) @@ -415,7 +437,34 @@ proc teaish__configure-phase1 {} { proj-dot-ins-append $dSrc/Makefile.in proj-dot-ins-append $dSrc/teaish.tester.tcl.in - define TEAISH_ENABLE_DIST [expr {![get-define TEAISH_OUT_OF_EXT_TREE]}] + apply {{} { + set vs [get-define TEAISH_VSATISFIES ""] + if {"" eq $vs} return + # Treat as a list-of-lists {{Tcl 8.5-} {Foo 1.0- -3.0} ...} and + # generate Tcl which will run package vsatisfies tests with that + # info. + set code {} + set n 0 + foreach pv $vs { + set n [llength $pv] + if {$n < 2} { + proj-error "-vsatisfies: {$pv} appears malformed. Whole list is: $vs" + } + set ifpv [list if \{ "!\[package vsatisfies \[package provide"] + lappend ifpv [lindex $pv 0] "\]" + for {set i 1} {$i < $n} {incr i} { + lappend ifpv [lindex $pv $i] + } + lappend ifpv "\] \} \{\n" + lappend ifpv \ + "error \{Package $::teaish__PkgInfo(-name) $::teaish__PkgInfo(-version) requires $pv\}" \ + "\n\}" + lappend code [join $ifpv] + } + define TEAISH_VSATISFIES_CODE [join $code "\n"] + }} + + define TEAISH_ENABLE_DIST $::teaish__Config(dist-enabled) define TEAISH_AUTOSETUP_DIR $::teaish__Config(core-dir) proj-setup-autoreconfig TEAISH_AUTORECONFIG foreach f { @@ -424,15 +473,16 @@ proc teaish__configure-phase1 {} { TEAISH_SRC TEAISH_DIST_FILES } { + # Ensure that any of these lists are flattened define $f [join [get-define $f]] } - proj-remap-autoconf-dir-vars define TEAISH__DEFINES_MAP \ [teaish__dump_defs_to_list]; # injected into teaish.tester.tcl + proj-remap-autoconf-dir-vars proj-dot-ins-process -validate; # do not [define] after this point proj-if-opt-truthy teaish-dump-defines { make-config-header config.defines.txt \ - -none {TEAISH__* TEAISH_MAKEFILE_CODE} \ + -none {TEAISH__* TEAISH_*_CODE} \ -str { BIN_* CC LD AR INSTALL LDFLAG* CFLAGS* *_LDFLAGS *_CFLAGS } \ @@ -765,6 +815,7 @@ If you are attempting an out-of-tree build, use set ::teaish__Config(teaish-dir) $dirExt set ::teaish__Config(blddir-is-extdir) \ [define TEAISH_ENABLE_DIST [expr {$dirBld eq $dirExt}]] + set ::teaish__Config(dist-enabled) $::teaish__Config(blddir-is-extdir); # may change later set addDist {{file} { teaish-dist-add [file tail $file] }} @@ -824,8 +875,6 @@ If you are attempting an out-of-tree build, use # Generate ./pkgIndex.tcl from it. define TEAISH_PKGINDEX_TCL_IN $extPI define TEAISH_PKGINDEX_TCL [file rootname [file tail $extPI]] - proj-dot-ins-append $extPI - file delete -force -- [get-define TEAISH_PKGINDEX_TCL] apply $addDist $extPI set piPolicy 0x01 } elseif {$dirExt ne $dirSrc @@ -833,17 +882,17 @@ If you are attempting an out-of-tree build, use # Generate ./pkgIndex.tcl from it. define TEAISH_PKGINDEX_TCL_IN $extPI define TEAISH_PKGINDEX_TCL [file rootname [file tail $extPI]] - proj-dot-ins-append $extPI - file delete -force -- [get-define TEAISH_PKGINDEX_TCL] set piPolicy 0x02 } elseif {[proj-first-file-found $dirExt/pkgIndex.tcl extPI]} { # Assume it's a static file and use it. define TEAISH_PKGINDEX_TCL_IN "" define TEAISH_PKGINDEX_TCL $extPI apply $addDist $extPI - set piPolicy 0x04 + set piPolicy 0x10 } - + # Reminder: we have to delay removal of stale TEAISH_PKGINDEX_TCL + # and the proj-dot-ins-append of TEAISH_PKGINDEX_TCL_IN until much + # later in the process. set ::teaish__Config(pkgindex-policy) $piPolicy # Look for teaish.test.tcl[.in] @@ -991,6 +1040,9 @@ proc teaish-src-add {args} { # proc teaish-dist-add {args} { if {$::teaish__Config(blddir-is-extdir)} { + # ^^^ reminder: we ignore $::teaish__Config(dist-enabled) here + # because the client might want to implement their own dist + # rules. proj-define-amend TEAISH_DIST_FILES {*}$args } } @@ -1045,8 +1097,7 @@ proc teaish-make-add {args} { # proc teaish-make-config-header {filename} { make-config-header $filename \ - -bare {} \ - -none {HAVE_CFLAG_* LDFLAGS_* SH_*} \ + -none {HAVE_CFLAG_* LDFLAGS_* SH_* TEAISH__* TEAISH_*_CODE} \ -auto {SIZEOF_* HAVE_* TEAISH_* TCL_*} \ -none * proj-touch $filename; # help avoid frequent unnecessary auto-reconfig @@ -1266,7 +1317,7 @@ proc teaish__dump_defs_to_list {args} { } # -# @teaish-pragma ...flags +# @teaish__pragma ...flags # # Offers a way to tweak how teaish's core behaves in some cases, in # particular those which require changing how the core looks for an @@ -1276,47 +1327,40 @@ proc teaish__dump_defs_to_list {args} { # during initial loading of tclish.tcl (recall that most teaish APIs # cannot be used until [teaish-configure] is called). # -# --have-own-pkgIndex.tcl [L]: Tells teaish that ./pkgIndex.tcl is -# not a generated file, so it will not try to overwrite or delete -# it. +# static-pkgIndex.tcl [L]: Tells teaish that ./pkgIndex.tcl is not +# a generated file, so it will not try to overwrite or delete +# it. Errors out if it does not find pkgIndex.tcl in the +# extension's dir. +# +# disable-dist [L]: tells teaish to elide the 'make dist' recipe +# from the generated Makefile. # # Emits a warning message for unknown arguments. # -proc teaish-pragma {args} { +proc teaish__pragma {args} { foreach arg $args { switch -exact -- $arg { - --have-own-pkgIndex.tcl { - set flist [list \ - [file join $::teaish__Config(teaish-dir) pkgIndex.tcl.in] \ - [file join $::teaish__Config(teaish-dir) pkgIndex.tcl]] - if {[proj-first-file-found $flist tpi]} { - if {[string match *.in $tpi]} { - define TEAISH_PKGINDEX_TCL_IN $tpi - teaish-dist-add [file tail $tpi] - define TEAISH_PKGINDEX_TCL [file rootname [file tail $pi]] - } else { - define TEAISH_PKGINDEX_TCL_IN "" - define TEAISH_PKGINDEX_TCL $tpi - teaish-dist-add [file tail $tpi] - } + static-pkgIndex.tcl { + set tpi [file join $::teaish__Config(teaish-dir) pkgIndex.tcl] + if {[file exists $tpi]} { + define TEAISH_PKGINDEX_TCL_IN "" + define TEAISH_PKGINDEX_TCL $tpi + set ::teaish__Config(pkgindex-policy) 0x20 } else { - proj-error "teaish-pragma $arg found no package-local pkgIndex.tcl\[.in]" + proj-error "$arg: found no package-local pkgIndex.tcl\[.in]" } - set ::teaish__Config(pkgindex-policy) 0x10 + } + + disable-dist { + define TEAISH_ENABLE_DIST 0 + set ::teaish__Config(dist-enabled) 0 } default { - proj-warn "Unknown [proj-current-scope] flag: $arg" + proj-error "Unknown flag: $arg" } } - -# --disable-dist [L]: disables the "dist" parts of the filtered -# Makefile. May be used during initial loading of teaish.tcl. -# -# --disable-dist { -# define TEAISH_ENABLE_DIST 0 -# } } } @@ -1334,31 +1378,43 @@ proc teaish-pragma {args} { # -libDir TEAISH_LIBDIR_NAME # -loadPrefix TEAISH_LOAD_PREFIX # -version TEAISH_VERSION -# -vsatisfies TEAISH_VSATISFIES_TCL +# -vsatisfies TEAISH_VSATISFIES # -options {...} optional [options-add] value # proc teaish-pkginfo-set {args} { set sentinel "" - set f2d $::teaish__Config(pkginfo-f2d) set flagDefs [list] - foreach {f d} $f2d { + foreach {f d} $::teaish__Config(pkginfo-f2d) { lappend flagDefs $f => $sentinel } proj-parse-simple-flags args flags $flagDefs if {[llength $args]} { proj-error -up "Too many (or unknown) arguments to [proj-current-scope]: $args" } - foreach {f d} $f2d { + foreach {f d} $::teaish__Config(pkginfo-f2d) { if {$sentinel ne [set v $flags($f)]} { switch -exact -- $f { -options { + proj-assert {"" eq $d} options-add $v } + -pragmas { + foreach p $v { + teaish__pragma $p + } + } + -vsatisfies { + if {1 == [llength $v] && 1 == [llength [lindex $v 0]]} { + # Transform X to {Tcl $X} + set v [list [join [list Tcl $v]]] + } + define $d $v + } default { define $d $v } } - set ::teaish__Config($f) $v + set ::teaish__PkgInfo($f) $v } } } @@ -1385,8 +1441,8 @@ proc teaish-pkginfo-get {args} { 0 { # Return a list of (-flag value) pairs lappend cases default {{ - if {[info exists ::teaish__Config($flag)]} { - lappend rv $flag $::teaish__Config($flag) + if {[info exists ::teaish__PkgInfo($flag)]} { + lappend rv $flag $::teaish__PkgInfo($flag) } else { lappend rv $flag [get-define $defName] } @@ -1398,8 +1454,8 @@ proc teaish-pkginfo-get {args} { if {[string match -* $arg]} { # Return the corresponding -flag's value lappend cases $arg {{ - if {[info exists ::teaish__Config($flag)]} { - return $::teaish__Config($flag) + if {[info exists ::teaish__PkgInfo($flag)]} { + return $::teaish__PkgInfo($flag) } else { return [get-define $defName] } @@ -1409,8 +1465,8 @@ proc teaish-pkginfo-get {args} { upvar $arg tgt array set tgt {} lappend cases default {{ - if {[info exists ::teaish__Config($flag)]} { - set tgt($flag) $::teaish__Config($flag) + if {[info exists ::teaish__PkgInfo($flag)]} { + set tgt($flag) $::teaish__PkgInfo($flag) } else { set tgt($flag) [get-define $defName] } @@ -1429,23 +1485,6 @@ proc teaish-pkginfo-get {args} { if {0 == $argc} { return $rv } } -#proc teaish-pget {flag} { -# teaish-pkginfo-get $flag -#} - -# -# @teaish-enable-dist ?yes? -# -# Explicitly enables or disables the "dist" rules in the default -# Makefile.in. This is equivalent to defining TEAISH_ENABLE_DIST -# to $yes (which must be 0 or 1). -# -# By default, dist creation is enabled. -# -proc teaish-enable-dist {{yes 1}} { - define TEAISH_ENABLE_DIST $yes -} - # # @teaish-checks-queue -pre|-post args... # @@ -1483,6 +1522,46 @@ proc teaish-checks-run {flag} { set ::teaish__Config(queued-checks${flag}) {} } +# +# A general-purpose getter for various teaish state. Requires one +# flag, which determines its result value. Flags marked with [L] below +# are safe for using at load-time, before teaish-pkginfo-set is called +# +# -dir [L]: returns the extension's directory, which may differ from +# the teaish core dir or the build dir. +# +# -teaish-home [L]: the "home" dir of teaish itself, which may +# differ from the extension dir or build dir. +# +# -build-dir [L]: the build directory (typically the current working +# -dir). +# +# Any of the teaish-pkginfo-get/get flags: returns the same as +# teaish-pkginfo-get. Not safe for use until teaish-pkginfo-set has +# been called. +# +# Triggers an error if passed an unknown flag. +# +proc teaish-get {flag} { + switch -exact -- $flag { + -dir { + return $::teaish__Config(teaish-dir) + } + -teaish-home { + return $::autosetup(srcdir) + } + -build-dir { + return $::autosetup(builddir) + } + default { + if {[info exists ::teaish__PkgInfo($flag)]} { + return $::teaish__PkgInfo($flag) + } + } + } + proj-error "Unhandled flag: $flag" +} + # # Handles --teaish-create-extension=TARGET-DIR # @@ -1520,7 +1599,7 @@ proc teaish__create_extension {dir} { -version ${version} \ -loadPrefix $loadPrefix \ -libDir ${name}${version} - -vsatisfies 8.5- \ + -vsatisfies {{Tcl 8.5-}} \ -options { foo=1 => {Disable foo} } #proc teaish-options {} { diff --git a/autoconf/tea/autosetup/tester.tcl b/autoconf/tea/autosetup/tester.tcl index 199f64dafb..863c058f2c 100644 --- a/autoconf/tea/autosetup/tester.tcl +++ b/autoconf/tea/autosetup/tester.tcl @@ -29,6 +29,7 @@ # call). If $lvl would resolve to global scope "global scope" is # returned and if it would be negative then a string indicating such # is returned (as opposed to throwing an error). +# proc test-current-scope {{lvl 0}} { #uplevel [expr {$lvl + 1}] {lindex [info level 0] 0} set ilvl [info level] @@ -45,6 +46,7 @@ proc test-current-scope {{lvl 0}} { # @test-msg # # Emits all arugments to stdout. +# proc test-msg {args} { puts "$args" } diff --git a/autoconf/tea/pkgIndex.tcl.in b/autoconf/tea/pkgIndex.tcl.in index 6ed6bb68db..8c50b81cdf 100644 --- a/autoconf/tea/pkgIndex.tcl.in +++ b/autoconf/tea/pkgIndex.tcl.in @@ -6,10 +6,8 @@ # edit it. # # Adapted from https://core.tcl-lang.org/tcltls -@if TEAISH_VSATISFIES_TCL -if {![package vsatisfies [package provide Tcl] @TEAISH_VSATISFIES_TCL@]} { - error "Package @TEAISH_PKGNAME@ @TEAISH_VERSION@ requires Tcl @TEAISH_VSATISFIES_TCL@" -} +@if TEAISH_VSATISFIES_CODE +@TEAISH_VSATISFIES_CODE@ @endif if {[package vsatisfies [package provide Tcl] 9.0-]} { package ifneeded {@TEAISH_PKGNAME@} {@TEAISH_VERSION@} [list apply {{dir} { diff --git a/autoconf/tea/teaish.tcl b/autoconf/tea/teaish.tcl index eedd9bdaf4..45f22ab5b1 100644 --- a/autoconf/tea/teaish.tcl +++ b/autoconf/tea/teaish.tcl @@ -1,7 +1,7 @@ # Teaish configure script for the SQLite TCL extension apply {{} { - set version [proj-file-content -trim [get-define TEAISH_DIR]/../VERSION] + set version [proj-file-content -trim [teaish-get -dir]/../VERSION] proj-assert {[string match 3.*.* $version]} teaish-pkginfo-set \ -name sqlite \ @@ -9,7 +9,8 @@ apply {{} { -version $version \ -loadPrefix Sqlite3 \ -vsatisfies 8.6- \ - -libDir sqlite$version + -libDir sqlite$version \ + -pragmas {disable-dist} }} # @@ -69,7 +70,6 @@ proc teaish-options {} { # work needed for this extension. # proc teaish-configure {} { - teaish-enable-dist 0 use teaish/feature-tests set srcdir [get-define TEAISH_DIR] diff --git a/autoconf/tea/teaish.tester.tcl.in b/autoconf/tea/teaish.tester.tcl.in index 315d82350b..ea4d2a0666 100644 --- a/autoconf/tea/teaish.tester.tcl.in +++ b/autoconf/tea/teaish.tester.tcl.in @@ -5,10 +5,8 @@ # probably not edit it. # # This is the wrapper script invoked by teaish's "make test" recipe. -@if TEAISH_VSATISFIES_TCL -if {![package vsatisfies [package provide Tcl] @TEAISH_VSATISFIES_TCL@]} { - error "Package @TEAISH_PKGNAME@ @TEAISH_VERSION@ requires Tcl @TEAISH_VSATISFIES_TCL@" -} +@if TEAISH_VSATISFIES_CODE +@TEAISH_VSATISFIES_CODE@ @endif load [lindex $::argv 0] [lindex $::argv 1]; source [lindex $::argv 2]; # teaish/tester.tcl diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 2e272a3b52..4c4d876f8c 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -139,7 +139,7 @@ proc proj-assert {script {msg ""}} { if {"" eq $msg} { set msg $script } - proj-fatal "Assertion failed: $msg" + proj-fatal "Assertion failed in \[[proj-current-scope 1]\]: $msg" } } @@ -297,7 +297,7 @@ proc proj-search-for-header-dir {header args} { -dirs { set args [lassign $args - dirs] } -subdirs { set args [lassign $args - subdirs] } default { - proj-fatal "Unhandled argument: $args" + proj-error "Unhandled argument: $args" } } } @@ -2064,7 +2064,7 @@ proc proj-coalesce {args} { # -flag defaultValue {script} # # -flag => defaultValue -# -----^--^ (wiith spaces there!) +# -----^--^ (with spaces there!) # # Repeated for each flag. # @@ -2096,8 +2096,8 @@ proc proj-coalesce {args} { # This function assumes that each flag is unique, and using a flag # more than once behaves in a last-one-wins fashion. # -# Any $argv entries not described in $prototype are not treated -# as flags. +# Any argvName entries not described in $prototype are not treated as +# flags. # # Returns the number of flags it processed in $argvName. # @@ -2113,6 +2113,10 @@ proc proj-coalesce {args} { # After that $flags would contain {-foo 1 -bar {blah} -no-baz 2} # and $args would be {8 9 10}. # +# Potential TODOs: consider using lappend instead of set so that any +# given flag can be used more than once. Or add a syntax to indicate +# that. +# proc proj-parse-simple-flags {argvName tgtArrayName prototype} { upvar $argvName argv upvar $tgtArrayName tgt @@ -2121,26 +2125,28 @@ proc proj-parse-simple-flags {argvName tgtArrayName prototype} { array set consuming {} set n [llength $prototype] # Figure out what our flags are... - for {set i 0} {$i < $n} {} { + for {set i 0} {$i < $n} {incr i} { set k [lindex $prototype $i] #puts "**** #$i of $n k=$k" proj-assert {[string match -* $k]} \ - "Invalid flag value for [proj-current-scope]: $k" + "Invalid flag value: $k" set v "" set s "" - if {"=>" eq [lindex $prototype [expr {$i + 1}]]} { - incr i 2 - if {$i >= $n} { - proj-fatal "Missing argument for $k => flag" + switch -exact -- [lindex $prototype [expr {$i + 1}]] { + => { + incr i 2 + if {$i >= $n} { + proj-error "Missing argument for $k => flag" + } + set consuming($k) 1 + set v [lindex $prototype $i] + } + default { + set v [lindex $prototype [incr i]] + set s [lindex $prototype [incr i]] + set scripts($k) $s } - set consuming($k) 1 - set v [lindex $prototype $i] - } else { - set v [lindex $prototype [incr i]] - set s [lindex $prototype [incr i]] - set scripts($k) $s } - incr i #puts "**** #$i of $n k=$k v=$v s=$s" set dflt($k) $v } @@ -2160,7 +2166,7 @@ proc proj-parse-simple-flags {argvName tgtArrayName prototype} { } elseif {[info exists tgt($arg)]} { if {[info exists consuming($arg)]} { if {$i + 1 >= $n} { - proj-fatal "Missing argument for $arg flag" + proj-assert 0 {Cannot happen - bounds already checked} } set tgt($arg) [lindex $argv [incr i]] } elseif {"" eq $scripts($arg)} { diff --git a/manifest b/manifest index 1dda006842..e1e9998c04 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sextra\sconditional\sthat\swas\sinserted\searlier\stoday\sand\swhich\nis\sunreachable. -D 2025-04-21T23:44:55.916 +C Generic\scleanups\sand\sfixes\sin\sthe\steaish\sbuild. +D 2025-04-22T01:01:19.374 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -22,20 +22,20 @@ F autoconf/Makefile.msc 5bc67d3912444c40c6f96d003e5c90663e51abb83d204a520110b1b2 F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac -F autoconf/tea/Makefile.in 41159d167c3b1a9bab7253f268d1ad1dc71ab8a9a6ccc9744a313aba40249a6a +F autoconf/tea/Makefile.in 88fc65f68618f7ab7d78bded67105d0787e5d71ab183f51d80a9bc5dce2478ab F autoconf/tea/README.txt 656d4686c509d375f5988ff3deda94f65fe6cd8358cd55d1f1dcc7b6e2ff73aa F autoconf/tea/auto.def 81e2617cfb90d53c19b53b3ec632cd2893bf32f2e5dd272b1116fadf2ea86c2d F autoconf/tea/autosetup/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca -F autoconf/tea/autosetup/core.tcl dea4684df6c5765a8fa866b4a6d79c73ba413ad37bb7c70e1be3631cd9eb2831 +F autoconf/tea/autosetup/core.tcl 9b8415824dff052ce25ce9beed895357d6b7cd8ed1422c76ba9744c3d2f9e7c4 F autoconf/tea/autosetup/feature-tests.tcl 307cac35806817fc87bd0d92004364ee682c0e99f2ab505291500493ab6c7a5f -F autoconf/tea/autosetup/tester.tcl a201ef9e91dde182e73015d09f94a3c40dd534ce38642167ebfd5884801d1e5a +F autoconf/tea/autosetup/tester.tcl 236598490975a3c071ee753900f3775192b116cf0531c6c754d17e028ea3f880 F autoconf/tea/configure d0b12b984edca6030d1976375b80157ac78b5b90a5b4f0dcee39357f63f4a80b x F autoconf/tea/doc/sqlite3.n 9a97f4f717ceab73004ea412af7960625c1cb24b5c25e4ae4c8b5d8fa4300f4e F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 -F autoconf/tea/pkgIndex.tcl.in a41b98633e184703136914e588638423047eae71244a30061ccd8edf92498436 -F autoconf/tea/teaish.tcl 2bc7ca287baa076daca513ff9342c00f499b1cceb8302a93d4a6231b00b52504 +F autoconf/tea/pkgIndex.tcl.in 14e8265862e8901a613ef50101f4f813a56f837a63f54e2431b9df0e9bc663a0 +F autoconf/tea/teaish.tcl c8e63cfb1b5cc5f688a90fea16c9bf8b0d6444dce62bd03f4abf7e8cdabf84ce F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 -F autoconf/tea/teaish.tester.tcl.in 63059e35289ac663c7d0052e6c0089a309fee75225e86e4ec5b3d9f2c1d9290a +F autoconf/tea/teaish.tester.tcl.in 9251c1beb38c24f708a35d74f75a0a1686cc8761034f99d8092282ac10040976 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e F autosetup/README.md f324bb9f9bf1cc787122034df53fbfdfed28ee2657e6652b763d992ab0d04829 @@ -51,7 +51,7 @@ F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 11e46ff237ca96d07fd14fe96132f8e77fee6442d8baa2e808bbe95016583c56 +F autosetup/proj.tcl 3eb8e6c74002fc89a1543406039bc36a9347e6deda90fadd00a7436820d38014 F autosetup/sqlite-config.tcl bde169c42d5d5331485ae7785f5dd19ca1aa01315787e8306441c472ad4e4201 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6538813cb89f6109727481e29633e2e98f98e0257c58695e3b53e8ce237d9195 -R f846b0d70084164be308ce186bb3a97a -U drh -Z 29505047856922d4c8d25e24a8f84ffb +P 5ea56af2d27611c5a8ea275316d21a02e70032f93243549ae776e9a89edc6229 +R d6b619df788366d9b93ebd1a1771f590 +U stephan +Z ae25c7334f797c5472e3ad844051b2c6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f8c99133e6..dd72c4a440 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ea56af2d27611c5a8ea275316d21a02e70032f93243549ae776e9a89edc6229 +734823a5a8e073b511ebc3f37df3542c1bb734b9b1e1bd50ca3e9df12c4bb89e From 024818be2b4a6b8e26866241adf6ad20aadaaeca Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 22 Apr 2025 09:29:53 +0000 Subject: [PATCH 05/42] Latest upstream jimtcl to fix rare segfault cases. FossilOrigin-Name: b6c5516940851bd17706435148253325d1b912a44ae138d2b8ca33e5167418c7 --- autosetup/jimsh0.c | 37 +++++++++++++++++++++++++------------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/autosetup/jimsh0.c b/autosetup/jimsh0.c index 1a6453d0c8..b035524c96 100644 --- a/autosetup/jimsh0.c +++ b/autosetup/jimsh0.c @@ -9132,7 +9132,7 @@ int Jim_StringEqObj(Jim_Obj *aObjPtr, Jim_Obj *bObjPtr) const char *sA = Jim_GetString(aObjPtr, &Alen); const char *sB = Jim_GetString(bObjPtr, &Blen); - return Alen == Blen && memcmp(sA, sB, Alen) == 0; + return Alen == Blen && *sA == *sB && memcmp(sA, sB, Alen) == 0; } } @@ -10242,7 +10242,7 @@ static int JimCommandsHT_KeyCompare(void *privdata, const void *key1, const void int len1, len2; const char *str1 = Jim_GetStringNoQualifier((Jim_Obj *)key1, &len1); const char *str2 = Jim_GetStringNoQualifier((Jim_Obj *)key2, &len2); - return len1 == len2 && memcmp(str1, str2, len1) == 0; + return len1 == len2 && *str1 == *str2 && memcmp(str1, str2, len1) == 0; } static void JimCommandsHT_ValDestructor(void *interp, void *val) @@ -13864,6 +13864,13 @@ static int JimExprOpNumUnary(Jim_Interp *interp, struct JimExprNode *node) case JIM_EXPROP_NOT: wC = !bA; break; + case JIM_EXPROP_UNARYPLUS: + case JIM_EXPROP_UNARYMINUS: + rc = JIM_ERR; + Jim_SetResultFormatted(interp, + "can't use non-numeric string as operand of \"%s\"", + node->type == JIM_EXPROP_UNARYPLUS ? "+" : "-"); + break; default: abort(); } @@ -19868,16 +19875,22 @@ wrongargs: } else if (errorCodeObj) { int len = Jim_ListLength(interp, argv[idx + 1]); - int i; - ret = JIM_OK; + if (len > Jim_ListLength(interp, errorCodeObj)) { - for (i = 0; i < len; i++) { - Jim_Obj *matchObj = Jim_ListGetIndex(interp, argv[idx + 1], i); - Jim_Obj *objPtr = Jim_ListGetIndex(interp, errorCodeObj, i); - if (Jim_StringCompareObj(interp, matchObj, objPtr, 0) != 0) { - ret = -1; - break; + ret = -1; + } + else { + int i; + ret = JIM_OK; + + for (i = 0; i < len; i++) { + Jim_Obj *matchObj = Jim_ListGetIndex(interp, argv[idx + 1], i); + Jim_Obj *objPtr = Jim_ListGetIndex(interp, errorCodeObj, i); + if (Jim_StringCompareObj(interp, matchObj, objPtr, 0) != 0) { + ret = -1; + break; + } } } } @@ -20253,7 +20266,7 @@ static int Jim_DictCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *arg } case OPT_SET: - return Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 4, argv[argc - 1], JIM_ERRMSG); + return Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 4, argv[argc - 1], JIM_ERRMSG | JIM_UNSHARED); case OPT_EXISTS:{ int rc = Jim_DictKeysVector(interp, argv[2], argv + 3, argc - 3, &objPtr, JIM_NONE); @@ -20265,7 +20278,7 @@ static int Jim_DictCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *arg } case OPT_UNSET: - if (Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 3, NULL, JIM_NONE) != JIM_OK) { + if (Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 3, NULL, JIM_UNSHARED) != JIM_OK) { return JIM_ERR; } return JIM_OK; diff --git a/manifest b/manifest index e1e9998c04..60c5b0ff3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Generic\scleanups\sand\sfixes\sin\sthe\steaish\sbuild. -D 2025-04-22T01:01:19.374 +C Latest\supstream\sjimtcl\sto\sfix\srare\ssegfault\scases. +D 2025-04-22T09:29:53.190 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -49,7 +49,7 @@ F autosetup/cc-lib.tcl 493c5935b5dd3bf9bd4eca89b07c8b1b1a9356d61783035144e21795f F autosetup/cc-shared.tcl 4f024e94a47f427ba61de1739f6381ef0080210f9fae89112d5c1de1e5460d78 F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e -F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 +F autosetup/jimsh0.c 563b966c137a4ce3c9333e5196723b7ac0919140a9d7989eb440463cd855c367 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba F autosetup/proj.tcl 3eb8e6c74002fc89a1543406039bc36a9347e6deda90fadd00a7436820d38014 F autosetup/sqlite-config.tcl bde169c42d5d5331485ae7785f5dd19ca1aa01315787e8306441c472ad4e4201 @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5ea56af2d27611c5a8ea275316d21a02e70032f93243549ae776e9a89edc6229 -R d6b619df788366d9b93ebd1a1771f590 +P 734823a5a8e073b511ebc3f37df3542c1bb734b9b1e1bd50ca3e9df12c4bb89e +R 7314569e2f49380ba2ad4032f6be829d U stephan -Z ae25c7334f797c5472e3ad844051b2c6 +Z 5242a8cac856aa0cbd261ef3d21cb581 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dd72c4a440..639296a283 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -734823a5a8e073b511ebc3f37df3542c1bb734b9b1e1bd50ca3e9df12c4bb89e +b6c5516940851bd17706435148253325d1b912a44ae138d2b8ca33e5167418c7 From 37f22ff75e37eab11c1fd8b04a8b59e75d655ef2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 22 Apr 2025 18:45:40 +0000 Subject: [PATCH 06/42] Fix a problem in the fts5matchinfo.test file that prevented it from working on Mac. FossilOrigin-Name: 5a411c609ba0ee17b1ce5bd0b40b9ded92d76f35933f92b2e02a58a3f169c715 --- ext/fts5/test/fts5matchinfo.test | 8 +++++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test index f81b076c18..a3bce869fb 100644 --- a/ext/fts5/test/fts5matchinfo.test +++ b/ext/fts5/test/fts5matchinfo.test @@ -528,9 +528,12 @@ do_execsql_test 16.0 { BEGIN EXCLUSIVE; } -sqlite3 db2 test.db do_test 16.1 { - catchsql { SELECT * FROM t1 } db2 + set rc [catch { + sqlite3 db2 test.db + db2 eval {SELECT * FROM t1} + } errmsg] + lappend rc $errmsg } {1 {database is locked}} do_execsql_test 16.2 { @@ -542,4 +545,3 @@ do_test 16.3 { } {0 {}} finish_test - diff --git a/manifest b/manifest index 60c5b0ff3e..73a3ec1ae6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Latest\supstream\sjimtcl\sto\sfix\srare\ssegfault\scases. -D 2025-04-22T09:29:53.190 +C Fix\sa\sproblem\sin\sthe\sfts5matchinfo.test\sfile\sthat\sprevented\sit\sfrom\nworking\son\sMac. +D 2025-04-22T18:45:40.700 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -205,7 +205,7 @@ F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c F ext/fts5/test/fts5locale.test 83ba7ee12628b540d3098f39c39c1de0c0440eddff8f7512c8c698d0c4a3ae3c -F ext/fts5/test/fts5matchinfo.test 7806f6d521bb49bcb54fff88a50f137866f7000c96ccfd28500caa47b63cb0aa +F ext/fts5/test/fts5matchinfo.test bc9e74157773db7f00aec1e85587f1145956ebdf1672c136f0f04323b2752aa0 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 F ext/fts5/test/fts5misc.test f4dee7da898d605a6488c5b7afaace3158ed6bb9addff78faa1b37b402b77fb9 @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 734823a5a8e073b511ebc3f37df3542c1bb734b9b1e1bd50ca3e9df12c4bb89e -R 7314569e2f49380ba2ad4032f6be829d -U stephan -Z 5242a8cac856aa0cbd261ef3d21cb581 +P b6c5516940851bd17706435148253325d1b912a44ae138d2b8ca33e5167418c7 +R 9218ccd8b518b209e6a20d61b383e872 +U drh +Z f58d140defc6379d7662d9d02b260125 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 639296a283..357f794a79 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b6c5516940851bd17706435148253325d1b912a44ae138d2b8ca33e5167418c7 +5a411c609ba0ee17b1ce5bd0b40b9ded92d76f35933f92b2e02a58a3f169c715 From d3b39b18d09df4c780e72190f68d11b776bcc4a7 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 23 Apr 2025 14:54:13 +0000 Subject: [PATCH 07/42] Upstream teaish fixes and improvements. FossilOrigin-Name: 235ea2c0e1823fe4324c3890069a9131ebb6de14e55eb56c0e9f42a7e491a699 --- autoconf/tea/Makefile.in | 42 ++--- autoconf/tea/auto.def | 5 +- autoconf/tea/autosetup/core.tcl | 287 ++++++++++++++++++++++-------- autoconf/tea/autosetup/tester.tcl | 16 +- autoconf/tea/pkgIndex.tcl.in | 4 + autoconf/tea/teaish.tcl | 22 +-- autoconf/tea/teaish.tester.tcl.in | 12 +- autosetup/proj.tcl | 46 +++-- autosetup/sqlite-config.tcl | 2 +- manifest | 30 ++-- manifest.uuid | 2 +- 11 files changed, 315 insertions(+), 153 deletions(-) diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index 172e381951..60c8761d62 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -46,7 +46,7 @@ tx.dll9.basename = @TEAISH_DLL9_BASENAME@ tx.dll8 = @TEAISH_DLL8@ tx.dll9 = @TEAISH_DLL9@ tx.dll = $(tx.dll$(TCL_MAJOR_VERSION)) -tx.dir = @TEAISH_DIR@ +tx.dir = @TEAISH_EXT_DIR@ teaish.dir = @abs_top_srcdir@ #teaish.dir.autosetup = @TEAISH_AUTOSETUP_DIR@ @@ -123,7 +123,7 @@ LDFLAGS.configure = @SH_LDFLAGS@ @TEAISH_LDFLAGS@ @LDFLAGS@ $(TCL_STUB_LIB_SPEC) # sources passed to [teaish-src-add], but may also be appended to # by teaish.make. # -tx.src =@TEAISH_SRC@ +tx.src =@TEAISH_EXT_SRC@ # # tx.CFLAGS is typically set by teaish.make, whereas TEAISH_CFLAGS @@ -161,7 +161,6 @@ teaish__autogen.deps = \ $(tx.makefile.in) $(teaish.makefile.in) \ $(tx.tcl) \ @TEAISH_PKGINDEX_TCL_IN@ \ - @TEAISH_MODULE_TEST_TCL@ \ @AUTODEPS@ # @@ -184,6 +183,7 @@ $(teaish.makefile): $(teaish__auto.def) $(teaish.makefile.in) \ @TEAISH_TESTER_TCL@: @TEAISH_TESTER_TCL_IN@ config.log: @TEAISH_TESTER_TCL@ +@if TEAISH_ENABLE_DLL # # The rest of this makefile exists solely to support this brief # target: the extension shared lib. @@ -193,6 +193,7 @@ $(tx.dll): $(tx.src) config.log $(tx.src) $(LDFLAGS.configure) $(LDFLAGS) $(tx.LDFLAGS) all: $(tx.dll) +@endif # TEAISH_ENABLE_DLL tclsh: $(teaish.makefile) config.log @{ echo "#!/bin/sh"; echo 'exec $(TCLSH) "$$@"'; } > $@ @@ -200,23 +201,24 @@ tclsh: $(teaish.makefile) config.log @echo "Created $@" # -# If the extension includes teaish.test.tcl then provide a "test" -# target which which runs that script, passing it (1) the full path to -# extension's DLL (which also provides the script with a way to get -# the test directory) and (2) a script of test utility code intended for -# sourcing by the client. +# Run the generated test script. # -# If the extension has no test script, add a small one which -# simply loads the DLL and success if it can. -# -# -tx.tester.args = $(tx.dll) $(tx.loadPrefix) @TEAISH_MODULE_TEST_TCL@ -.PHONY: test-pre test-core test test-post test-extension -test-extension: # this name is reserved for use by teaish.make -test-prepre: $(tx.dll) @TEAISH_TESTER_TCL@ +.PHONY: test-pre test-prepre test-core test test-post test-extension +test-extension: # this name is reserved for use by teaish.make[.in] +@if TEAISH_ENABLE_DLL +test-prepre: $(tx.dll) +@endif +test-core.args = @TEAISH_TESTER_TCL@ +@if TEAISH_ENABLE_DLL +test-core.args += '$(tx.dll)' '$(tx.loadPrefix)' +@else +test-core.args += '' '' +@endif +test-core.args += @TEAISH_TESTUTIL_TCL@ +test-prepre: @TEAISH_TESTER_TCL@ test-pre: test-prepre test-core: test-pre - $(TCLSH) @TEAISH_TESTER_TCL@ $(tx.tester.args) + $(TCLSH) $(test-core.args) test-post: test-core test: test-post @@ -367,12 +369,12 @@ undist-tgz: rm -f $(dist.tgz) undist: undist-tgz @endif #BIN_TAR -@else +@else #!TEAISH_ENABLE_DIST undist: dist: @if TEAISH_OUT_OF_EXT_TREE @echo "'dist' can only be used from an extension's home dir" 1>&2; \ - echo "In this case: @TEAISH_DIR@" 1>&2; exit 1 + echo "In this case: @TEAISH_EXT_DIR@" 1>&2; exit 1 @endif @endif #TEAISH_ENABLE_DIST @@ -386,7 +388,7 @@ dist: # - tx.src = list of the extension's source files, being sure to # prefix each with $(tx.dir) (if it's in the same dir as the # extension) so that out-of-tree builds can find them. Optionally, -# [define] TEAISH_SRC or pass them to [teaish-src-add]. +# [define] TEAISH_EXT_SRC or pass them to [teaish-src-add]. # # It may optionally set the following vars: # diff --git a/autoconf/tea/auto.def b/autoconf/tea/auto.def index 861257cce3..7170b3d1fe 100644 --- a/autoconf/tea/auto.def +++ b/autoconf/tea/auto.def @@ -1,7 +1,8 @@ #/do/not/tclsh # ^^^ help out editors which guess this file's content type. # -# Main configure script entry point for the "TEA-via-autosetup" -# framework. +# Main configure script entry point for the TEA(ish) framework. All +# extension-specific customization goes in teaish.tcl.in or +# teaish.tcl. use teaish/core teaish-configure-core diff --git a/autoconf/tea/autosetup/core.tcl b/autoconf/tea/autosetup/core.tcl index 14044e607a..9ef0178e29 100644 --- a/autoconf/tea/autosetup/core.tcl +++ b/autoconf/tea/autosetup/core.tcl @@ -23,18 +23,22 @@ use proj -define TEAISH_CORE_VERSION 0.1-beta - # # API-internal settings and shared state. array set teaish__Config [proj-strip-hash-comments { + # + # Teaish's version number, not to be confused with + # teaish__PkgInfo(-version). + # + version 0.1-beta + # set to 1 to enable some internal debugging output debug-enabled 0 # # 0 = don't yet have extension's pkgindex - # 0x01 = found TEAISH_DIR/pkgIndex.tcl.in + # 0x01 = found TEAISH_EXT_DIR/pkgIndex.tcl.in # 0x02 = found srcdir/pkgIndex.tcl.in - # 0x10 = found TEAISH_DIR/pkgIndex.tcl (static file) + # 0x10 = found TEAISH_EXT_DIR/pkgIndex.tcl (static file) # 0x20 = static-pkgIndex.tcl pragma: behave as if 0x10 # # Reminder: it's significant that the bottom 4 bits be @@ -53,9 +57,9 @@ array set teaish__Config [proj-strip-hash-comments { pkginfo-f2d { -name TEAISH_NAME -pkgName TEAISH_PKGNAME + -version TEAISH_VERSION -libDir TEAISH_LIBDIR_NAME -loadPrefix TEAISH_LOAD_PREFIX - -version TEAISH_VERSION -vsatisfies TEAISH_VSATISFIES -options {} -pragmas {} @@ -71,6 +75,24 @@ array set teaish__Config [proj-strip-hash-comments { # when building from an extension's dir, disabled when building # elsewhere. dist-enabled 1 + + # By default we enable compilation of a native extension but if the + # extension has no native code or the user wants to take that over + # via teaish.make.in or provide a script-only extension, we will + # elide the default compilation rules if this is 0. + dll-enabled 1 + + # Files to include in the "make dist" bundle. + dist-files {} + + # List of source files for the extension. + extension-src {} + + # Path to the teaish.tcl file. + teaish.tcl {} + + # Dir where teaish.tcl is found. + extension-dir {} }] set teaish__Config(core-dir) $::autosetup(libdir)/teaish @@ -116,7 +138,7 @@ if {[teaish-argv-has --teaish-verbose --t-v]} { msg-quiet use system ; # Outputs "Host System" and "Build System" lines if {"--help" ni $::argv} { - teaish__verbose 1 msg-result "TEA(ish) Version = [get-define TEAISH_CORE_VERSION]" + teaish__verbose 1 msg-result "TEA(ish) Version = $::teaish__Config(version)" teaish__verbose 1 msg-result "Source dir = $::autosetup(srcdir)" teaish__verbose 1 msg-result "Build dir = $::autosetup(builddir)" } @@ -194,16 +216,15 @@ proc teaish-configure-core {} { }]; # main options. if {$gotExt} { - set ttcl [get-define TEAISH_TCL] - proj-assert {[file exists $ttcl]} "Expecting to have found teaish.tcl by now" - uplevel 1 [list source $ttcl] proj-assert {"" ne [teaish-pkginfo-get -name]} - unset ttcl + proj-assert {[file exists $::teaish__Config(teaish.tcl)]} \ + "Expecting to have found teaish.tcl by now" + uplevel 1 [list source $::teaish__Config(teaish.tcl)] # Set up some default values if the extension did not set them. - # This must happen _after_ it's sourced. + # This must happen _after_ it's sourced but before + # teaish-configure is called. foreach {pflag key type val} { - TEAISH_CFLAGS -v "" - - TEAISH_DIST_FILES -v "" - TEAISH_LDFLAGS -v "" - TEAISH_MAKEFILE -v "" - TEAISH_MAKEFILE_CODE -v "" @@ -212,16 +233,19 @@ proc teaish-configure-core {} { - TEAISH_PKGINDEX_TCL_IN -v "" - TEAISH_PKGINIT_TCL -v "" - TEAISH_PKGINIT_TCL_IN -v "" - - TEAISH_SRC -v "" - TEAISH_TEST_TCL -v "" - TEAISH_TEST_TCL_IN -v "" - -version TEAISH_VERSION -v 0.0.0 - -pkgName TEAISH_PKGNAME -e {teaish-pkginfo-get -name} - -libDir TEAISH_LIBDIR_NAME -e {join [list \ - [teaish-pkginfo-get -pkgName] \ - [teaish-pkginfo-get -version]] ""} - -loadPrefix TEAISH_LOAD_PREFIX -e {string totitle [get-define TEAISH_PKGNAME ""]} + -version TEAISH_VERSION -v 0.0.0 + -pkgName TEAISH_PKGNAME -e {teaish-pkginfo-get -name} + -libDir TEAISH_LIBDIR_NAME -e { + join [list \ + [teaish-pkginfo-get -pkgName] \ + [teaish-pkginfo-get -version]] "" + } + -loadPrefix TEAISH_LOAD_PREFIX -e { + string totitle [teaish-get -pkgName] + } -vsatisfies TEAISH_VSATISFIES -v {{Tcl 8.5-}} } { set isDefOnly [expr {"-" eq $pflag}] @@ -243,12 +267,12 @@ proc teaish-configure-core {} { set ::teaish__PkgInfo($pflag) $val } } - unset key type val + unset isDefOnly pflag key type val }; # sourcing extension's teaish.tcl if {[llength [info proc teaish-options]] > 0} { # Add options defined by teaish-options, which is assumed to be - # imported via TEAISH_TCL. + # imported via [teaish-get -teaish-tcl]. set o [teaish-options] if {"" ne $o} { options-add $o @@ -372,7 +396,7 @@ proc teaish__configure_phase1 {} { teaish-checks-run -pre if {[llength [info proc teaish-configure]] > 0} { # teaish-configure is assumed to be imported via - # TEAISH_TCL + # teaish.tcl teaish-configure } teaish-checks-run -post @@ -405,7 +429,7 @@ proc teaish__configure_phase1 {} { } #define AS_LIBDIR $::autosetup(libdir) - define TEAISH_MODULE_TEST_TCL $::teaish__Config(core-dir)/tester.tcl + define TEAISH_TESTUTIL_TCL $::teaish__Config(core-dir)/tester.tcl apply {{} { # @@ -431,7 +455,7 @@ proc teaish__configure_phase1 {} { } }}; # $::teaish__Config(pkgindex-policy) - set dEx $::teaish__Config(teaish-dir) + set dEx $::teaish__Config(extension-dir) set dSrc $::autosetup(srcdir) proj-dot-ins-append $dSrc/Makefile.in @@ -457,21 +481,24 @@ proc teaish__configure_phase1 {} { } lappend ifpv "\] \} \{\n" lappend ifpv \ - "error \{Package $::teaish__PkgInfo(-name) $::teaish__PkgInfo(-version) requires $pv\}" \ - "\n\}" + "error \{Package $::teaish__PkgInfo(-name) $::teaish__PkgInfo(-version) requires $pv\}\n" \ + "\}" lappend code [join $ifpv] } define TEAISH_VSATISFIES_CODE [join $code "\n"] }} - define TEAISH_ENABLE_DIST $::teaish__Config(dist-enabled) - define TEAISH_AUTOSETUP_DIR $::teaish__Config(core-dir) + define TEAISH_ENABLE_DIST $::teaish__Config(dist-enabled) + define TEAISH_ENABLE_DLL $::teaish__Config(dll-enabled) + define TEAISH_AUTOSETUP_DIR $::teaish__Config(core-dir) + define TEAISH_TCL $::teaish__Config(teaish.tcl) + define TEAISH_DIST_FILES [join $::teaish__Config(dist-files)] + define TEAISH_EXT_DIR [join $::teaish__Config(extension-dir)] + define TEAISH_EXT_SRC [join $::teaish__Config(extension-src)] proj-setup-autoreconfig TEAISH_AUTORECONFIG foreach f { TEAISH_CFLAGS TEAISH_LDFLAGS - TEAISH_SRC - TEAISH_DIST_FILES } { # Ensure that any of these lists are flattened define $f [join [get-define $f]] @@ -716,7 +743,7 @@ proc teaish__check_tcl {} { # This does not _load_ the extension, it simply locates the files # which make up an extension. # -# This sets up lots of defines, e.g. TEAISH_DIR. +# This sets up lots of defines, e.g. TEAISH_EXT_DIR. # proc teaish__find_extension {} { @@ -760,8 +787,7 @@ proc teaish__find_extension {} { proj-error "--teaish-extension-dir value is not a directory: $extD" } set extT [apply $lambdaMT 1 teaish.tcl $extD] - define TEAISH_DIR $extD - set ::teaish__Config(teaish-dir) $extD + set ::teaish__Config(extension-dir) $extD } --help { incr gotHelpArg @@ -774,7 +800,7 @@ proc teaish__find_extension {} { set ::argv $largv set dirExt [proj-coalesce \ - [get-define TEAISH_DIR ""] \ + $::teaish__Config(extension-dir) \ $dirBld]; # dir with the extension # # teaish.tcl is a TCL script which implements various @@ -803,7 +829,7 @@ If you are attempting an out-of-tree build, use if {![file readable $extT]} { proj-error "extension tcl file is not readable: $extT" } - define TEAISH_TCL $extT + set ::teaish__Config(teaish.tcl) $extT if {"" eq $dirExt} { # If this wasn't set via --teaish-extension-dir then derive it from @@ -811,17 +837,17 @@ If you are attempting an out-of-tree build, use #puts "extT=$extT dirExt=$dirExt" set dirExt [file dirname $extT] } - define TEAISH_DIR $dirExt - set ::teaish__Config(teaish-dir) $dirExt - set ::teaish__Config(blddir-is-extdir) \ - [define TEAISH_ENABLE_DIST [expr {$dirBld eq $dirExt}]] + + set ::teaish__Config(extension-dir) $dirExt + set ::teaish__Config(blddir-is-extdir) [expr {$dirBld eq $dirExt}] set ::teaish__Config(dist-enabled) $::teaish__Config(blddir-is-extdir); # may change later + set addDist {{file} { teaish-dist-add [file tail $file] }} apply $addDist $extT - teaish__verbose 1 msg-result "Extension dir = [get-define TEAISH_DIR]" + teaish__verbose 1 msg-result "Extension dir = [teaish-get -dir]" teaish__verbose 1 msg-result "Extension config = $extT" teaish-pkginfo-set -name [file tail [file dirname $extT]] @@ -939,10 +965,9 @@ If you are attempting an out-of-tree build, use # TEAISH_OUT_OF_EXT_TREE = 1 if we're building from a dir other # than the extension's home dir. - set dteaish [file-normalize [get-define TEAISH_DIR]] define TEAISH_OUT_OF_EXT_TREE \ - [expr {[file-normalize $::autosetup(builddir)] ne $dteaish}] - + [expr {[file-normalize $::autosetup(builddir)] ne \ + [file-normalize $::teaish__Config(extension-dir)]}] return 1 }; # teaish__find_extension @@ -994,56 +1019,57 @@ proc teaish-ldflags-prepend {args} { # # @teaish-src-add ?-dist? ?-dir? src-files... # -# Appends all non-empty $args to TEAISH_SRC. +# Appends all non-empty $args to the project's list of C/C++ source +# files. # # If passed -dist then it also passes each filename, as-is, to # [teaish-dist-add]. # -# If passed -dir then each src-file has the TEAISH_DIR prepended to -# it for before they're added to TEAISH_SRC. As often as not, that -# will be the desired behavior so that out-of-tree builds can find the +# If passed -dir then each src-file has [teaish-get -dir] prepended to +# it for before they're added to the list. As often as not, that will +# be the desired behavior so that out-of-tree builds can find the # sources, but there are cases where it's not desired (e.g. when using # a source file from outside of the extension's dir). # proc teaish-src-add {args} { set i 0 proj-parse-simple-flags args flags { - -dist 0 {return 1} - -dir 0 {return 1} + -dist 0 {expr 1} + -dir 0 {expr 1} } if {$flags(-dist)} { teaish-dist-add {*}$args } if {$flags(-dir)} { set xargs {} - set d [get-define TEAISH_DIR] foreach arg $args { if {"" ne $arg} { - lappend xargs [file join $d $arg] + lappend xargs [file join $::teaish__Config(extension-dir) $arg] } } set args $xargs } - proj-define-append TEAISH_SRC {*}$args + lappend ::teaish__Config(extension-src) {*}$args } # # @teaish-dist-add files-or-dirs... # -# Equivalent to [proj-define-apend TEAISH_DIST_FILES ...]. +# Adds the given files to the list of files to include with the "make +# dist" rules. # # This is a no-op when the current build is not in the extension's # directory, as dist support is disabled in out-of-tree builds. # -# It is not legal to call this until TEAISH_DIR has been reliably set -# (via teaish__find_extension). +# It is not legal to call this until [teaish-get -dir] has been +# reliably set (via teaish__find_extension). # proc teaish-dist-add {args} { if {$::teaish__Config(blddir-is-extdir)} { # ^^^ reminder: we ignore $::teaish__Config(dist-enabled) here # because the client might want to implement their own dist # rules. - proj-define-amend TEAISH_DIST_FILES {*}$args + lappend ::teaish__Config(dist-files) {*}$args } } @@ -1332,7 +1358,10 @@ proc teaish__dump_defs_to_list {args} { # it. Errors out if it does not find pkgIndex.tcl in the # extension's dir. # -# disable-dist [L]: tells teaish to elide the 'make dist' recipe +# no-dist [L]: tells teaish to elide the 'make dist' recipe +# from the generated Makefile. +# +# no-dll [L]: tells teaish to elide the DLL-building recipe # from the generated Makefile. # # Emits a warning message for unknown arguments. @@ -1342,7 +1371,7 @@ proc teaish__pragma {args} { switch -exact -- $arg { static-pkgIndex.tcl { - set tpi [file join $::teaish__Config(teaish-dir) pkgIndex.tcl] + set tpi [file join $::teaish__Config(extension-dir) pkgIndex.tcl] if {[file exists $tpi]} { define TEAISH_PKGINDEX_TCL_IN "" define TEAISH_PKGINDEX_TCL $tpi @@ -1352,11 +1381,14 @@ proc teaish__pragma {args} { } } - disable-dist { - define TEAISH_ENABLE_DIST 0 + no-dist { set ::teaish__Config(dist-enabled) 0 } + no-dll { + set ::teaish__Config(dll-enabled) 0 + } + default { proj-error "Unknown flag: $arg" } @@ -1367,21 +1399,121 @@ proc teaish__pragma {args} { # # @teaish-pkginfo-set ...flags # -# The preferred way to set up the initial package state. Used like: +# The way to set up the initial package state. Used like: # # teaish-pkginfo-set -name foo -version 0.1.2 # -# Where each flag corresponds to one piece of extension package info. +# Or: # -# -name TEAISH_NAME -# -pkgName TEAISH_PKGNAME -# -libDir TEAISH_LIBDIR_NAME -# -loadPrefix TEAISH_LOAD_PREFIX -# -version TEAISH_VERSION -# -vsatisfies TEAISH_VSATISFIES -# -options {...} optional [options-add] value +# teaish-pkginfo-set ?-vars|-subst? {-name foo -version 0.1.2} # +# The latter may be easier to write for a multi-line invocation. +# Passing the -vars flag tells it to perform a [subst] of (only) +# variables in the {...} part from the calling scope. The -subst flag +# will cause is to [subst] the {...} with command substitution as well +# (but no backslash substitution). +# +# Each pkginfo flag corresponds to one piece of extension package +# info. Teaish provides usable default values for all of these flags, +# but at least the -name and -version should be set by clients. +# e.g. the default -name is the directory name the extension lives in, +# which may change (e.g. when building it from a "make dist" bundle). +# +# The flags: +# +# -name theName: The extension's name. It defaults to the name of the +# directory containing the extension. (In TEA this would be the +# PACKAGE_NAME, not to be confused with...) +# +# -pkgName pkg-provide-name: The extension's name for purposes of +# Tcl_PkgProvide(), [package require], and friends. It defaults to +# the `-name`, and is normally the same, but some projects (like +# SQLite) have a different name here than they do in their +# historical TEA PACKAGE_NAME. +# +# -version version: The extension's package version. Defaults to +# 0.0.0. +# +# -libDir dirName: The base name of the directory into which this +# extension should be installed. It defaults to a concatenation of +# `-pkgName` and `-version`. +# +# -loadPrefix prefix: For use as the second argument passed to +# Tcl's `load` command in the package-loading process. It defaults +# to title-cased `-pkgName` because Tcl's `load` plugin system +# expects it in that form. +# +# -options {...}: If provided, it must be a list compatible with +# Autosetup's `options-add` function. These can also be set up via +# `teaish-options`. +# +# -vsatisfies {{...} ...}: Expects a list-of-lists of conditions +# for Tcl's `package vsatisfies` command: each list entry is a +# sub-list of `{PkgName Condition...}`. Teaish inserts those +# checks via its default pkgIndex.tcl.in and teaish.tester.tcl.in +# templates to verify that the system's package dependencies meet +# these requirements. The default value is `{{Tcl 8.5-}}` (recall +# that it's a list-of-lists), as 8.5 is the minimum Tcl version +# teaish will run on, but some extensions may require newer +# versions or dependencies on other packages. As a special case, +# if `-vsatisfies` is given a single token, e.g. `8.6-`, then it +# is transformed into `{Tcl $thatToken}`, i.e. it checks the Tcl +# version which the package is being run with. If given multiple +# lists, each `package provides` check is run in the given +# order. Failure to meet a `vsatisfies` condition triggers an +# error. +# +# -pragmas {...} A list of infrequently-needed lower-level +# directives which can influence teaish, including: +# +# static-pkgIndex.tcl: tells teaish that the client manages their +# own pkgIndex.tcl, so that teaish won't try to overwrite it +# using a template. +# +# no-dist: tells teaish to elide the "make dist" recipe from the +# makefile so that the client can implement it. +# +# no-dll: tells teaish to elide the makefile rules which build +# the DLL, as well as any templated test script and pkgIndex.tcl +# references to them. The intent here is to (A) support +# client-defined build rules for the DLL and (B) eventually +# support script-only extensions. +# +# Unsupported flags or pragmas will trigger an error. proc teaish-pkginfo-set {args} { + set doVars 0 + set doCommands 0 + set xargs $args + foreach arg $args { + switch -exact -- $arg { + -vars { + incr doVars + set xargs [lassign $xargs -] + } + -subst { + incr doVars + incr doCommands + set xargs [lassign $xargs -] + } + default { + break + } + } + } + set args $xargs + unset xargs + if {1 == [llength $args] && [llength [lindex $args 0]] > 1} { + # Transform a single {...} arg into the canonical call form + set a [list {*}[lindex $args 0]] + if {$doVars || $doCommands} { + set sflags -nobackslashes + if {!$doCommands} { + lappend sflags -nocommands + } + set a [uplevel 1 [list subst {*}$sflags $a]] + } + set args $a + } set sentinel "" set flagDefs [list] foreach {f d} $::teaish__Config(pkginfo-f2d) { @@ -1545,7 +1677,7 @@ proc teaish-checks-run {flag} { proc teaish-get {flag} { switch -exact -- $flag { -dir { - return $::teaish__Config(teaish-dir) + return $::teaish__Config(extension-dir) } -teaish-home { return $::autosetup(srcdir) @@ -1553,6 +1685,9 @@ proc teaish-get {flag} { -build-dir { return $::autosetup(builddir) } + -teaish.tcl { + return $::teaish__Config(teaish.tcl) + } default { if {[info exists ::teaish__PkgInfo($flag)]} { return $::teaish__PkgInfo($flag) @@ -1610,15 +1745,15 @@ proc teaish__create_extension {dir} { # early on may be used here. Any autosetup-related # APIs may be used here. # - # Return an empty string if there are no options to add - # or if they are added using \[options-add\]. + # Return an empty string if there are no options to + # add or if they are added using \[options-add\]. + # + # If there are no options to add, this proc need + # not be defined. #} proc teaish-configure {} { - set d \[get-define TEAISH_DIR\] - teaish-src-add \$d/teaish.c - teaish-dist-add teaish.c - + teaish-src-add -dir -dist teaish.c # TODO: your code goes here.. } " diff --git a/autoconf/tea/autosetup/tester.tcl b/autoconf/tea/autosetup/tester.tcl index 863c058f2c..5f5e1f1555 100644 --- a/autoconf/tea/autosetup/tester.tcl +++ b/autoconf/tea/autosetup/tester.tcl @@ -144,7 +144,11 @@ if {![array exists ::teaish__BuildFlags]} { } # -# @teaish-build-flag2 flag tgtVar ?dflt? +# @teaish-build-flag3 flag tgtVar ?dflt? +# +# If the current build has the configure-time flag named $flag set +# then tgtVar is assigned its value and 1 is returned, else tgtVal is +# assigned $dflt and 0 is returned. # # Caveat #1: only valid when called in the context of teaish's default # "make test" recipe, e.g. from teaish.test.tcl. It is not valid from @@ -154,15 +158,11 @@ if {![array exists ::teaish__BuildFlags]} { # an external script have populated its internal state, which is # normally handled via teaish.tester.tcl.in. # -# If the current build has the configure-time flag named $flag set -# then tgtVar is assigned its value and 1 is returned, else tgtVal is -# assigned $dflt and 0 is returned. -# # Caveat #2: defines in the style of HAVE_FEATURENAME with a value of # 0 are, by long-standing configure script conventions, treated as # _undefined_ here. # -proc teaish-build-flag2 {flag tgtVar {dflt ""}} { +proc teaish-build-flag3 {flag tgtVar {dflt ""}} { upvar $tgtVar tgt if {[info exists ::teaish__BuildFlags($flag)]} { set tgt $::teaish__BuildFlags($flag) @@ -179,12 +179,12 @@ proc teaish-build-flag2 {flag tgtVar {dflt ""}} { # # @teaish-build-flag flag ?dflt? # -# Convenience form of teaish-build-flag2 which returns the +# Convenience form of teaish-build-flag3 which returns the # configure-time-defined value of $flag or "" if it's not defined (or # if it's an empty string). # proc teaish-build-flag {flag {dflt ""}} { set tgt "" - teaish-build-flag2 $flag tgt $dflt + teaish-build-flag3 $flag tgt $dflt return $tgt } diff --git a/autoconf/tea/pkgIndex.tcl.in b/autoconf/tea/pkgIndex.tcl.in index 8c50b81cdf..c93fcc6854 100644 --- a/autoconf/tea/pkgIndex.tcl.in +++ b/autoconf/tea/pkgIndex.tcl.in @@ -11,7 +11,9 @@ @endif if {[package vsatisfies [package provide Tcl] 9.0-]} { package ifneeded {@TEAISH_PKGNAME@} {@TEAISH_VERSION@} [list apply {{dir} { +@if TEAISH_ENABLE_DLL load [file join $dir {@TEAISH_DLL9@}] @TEAISH_LOAD_PREFIX@ +@endif @if TEAISH_PKGINIT_TCL_TAIL set initScript [file join $dir {@TEAISH_PKGINIT_TCL_TAIL@}] if {[file exists $initScript]} { @@ -21,11 +23,13 @@ if {[package vsatisfies [package provide Tcl] 9.0-]} { }} $dir] } else { package ifneeded {@TEAISH_PKGNAME@} {@TEAISH_VERSION@} [list apply {{dir} { +@if TEAISH_ENABLE_DLL if {[string tolower [file extension {@TEAISH_DLL8@}]] in [list .dll .dylib .so]} { load [file join $dir {@TEAISH_DLL8@}] @TEAISH_LOAD_PREFIX@ } else { load {} @TEAISH_LOAD_PREFIX@ } +@endif @if TEAISH_PKGINIT_TCL_TAIL set initScript [file join $dir {@TEAISH_PKGINIT_TCL_TAIL@}] if {[file exists $initScript]} { diff --git a/autoconf/tea/teaish.tcl b/autoconf/tea/teaish.tcl index 45f22ab5b1..53228cb910 100644 --- a/autoconf/tea/teaish.tcl +++ b/autoconf/tea/teaish.tcl @@ -3,14 +3,15 @@ apply {{} { set version [proj-file-content -trim [teaish-get -dir]/../VERSION] proj-assert {[string match 3.*.* $version]} - teaish-pkginfo-set \ - -name sqlite \ - -pkgName sqlite3 \ - -version $version \ - -loadPrefix Sqlite3 \ - -vsatisfies 8.6- \ - -libDir sqlite$version \ - -pragmas {disable-dist} + teaish-pkginfo-set -vars { + -name sqlite + -pkgName sqlite3 + -version $version + -loadPrefix Sqlite3 + -vsatisfies 8.6- + -libDir sqlite$version + -pragmas {no-dist} + } }} # @@ -72,9 +73,8 @@ proc teaish-options {} { proc teaish-configure {} { use teaish/feature-tests - set srcdir [get-define TEAISH_DIR] + set srcdir [teaish-get -dir] teaish-src-add -dist -dir generic/tclsqlite3.c - teaish-cflags-add -I${srcdir}/.. if {[proj-opt-was-provided override-sqlite-version]} { teaish-pkginfo-set -version [opt-val override-sqlite-version] proj-warn "overriding sqlite version number:" [teaish-pkginfo-get -version] @@ -103,6 +103,8 @@ proc teaish-configure {} { msg-result "Using system-level sqlite3." teaish-cflags-add -DUSE_SYSTEM_SQLITE teaish-ldflags-add -lsqlite3 + } else { + teaish-cflags-add -I${srcdir}/.. } teaish-check-librt diff --git a/autoconf/tea/teaish.tester.tcl.in b/autoconf/tea/teaish.tester.tcl.in index ea4d2a0666..8f1362a75c 100644 --- a/autoconf/tea/teaish.tester.tcl.in +++ b/autoconf/tea/teaish.tester.tcl.in @@ -5,10 +5,20 @@ # probably not edit it. # # This is the wrapper script invoked by teaish's "make test" recipe. +# It gets passed 3 args: +# +# $1 = the DLL name, or "" if the extension has no DLL +# +# $2 = the "load prefix" for Tcl's [load] or empty if $1 is empty +# +# $3 = the /path/to/teaish/tester.tcl (test utility code) +# @if TEAISH_VSATISFIES_CODE @TEAISH_VSATISFIES_CODE@ @endif -load [lindex $::argv 0] [lindex $::argv 1]; +if {[llength [lindex $::argv 0]] > 0} { + load [lindex $::argv 0] [lindex $::argv 1]; +} source [lindex $::argv 2]; # teaish/tester.tcl @if TEAISH_PKGINIT_TCL apply {{file} { diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 4c4d876f8c..1ccc632809 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -8,7 +8,13 @@ # * May you find forgiveness for yourself and forgive others. # * May you share freely, never taking more than you give. # -######################################################################## + +# +# ----- @module proj.tcl ----- +# @section Project Helper APIs +# + +# # Routines for Steve Bennett's autosetup which are common to trees # managed in and around the umbrella of the SQLite project. # @@ -25,13 +31,12 @@ # noted here only as an indication that there are no licensing issues # despite this code having a handful of near-twins running around a # handful of third-party source trees. -######################################################################## # # Design notes: # -# - Symbols with a suffix of _ are intended for internal use within +# - Symbols with _ separators are intended for internal use within # this file, and are not part of the API which auto.def files should -# rely on. +# rely on. Symbols with - separators are public APIs. # # - By and large, autosetup prefers to update global state with the # results of feature checks, e.g. whether the compiler supports flag @@ -49,10 +54,7 @@ # test, downstream tests may not like the $prefix/lib path added by # the rpath test. To avoid such problems, we avoid (intentionally) # updating global state via feature tests. -######################################################################## - -# ----- @module proj.tcl ----- -# @section Project Helper APIs +# # # $proj__Config is an internal-use-only array for storing whatever generic @@ -1427,11 +1429,11 @@ proc proj-which-linenoise {dotH} { # manner unless they are explicitly overridden at configure-time, in # which case those overrides takes precedence. # -# Each --XYZdir flag which is explicitly passed to configure is -# exported as-is, as are those which default to some top-level system -# directory, e.g. /etc or /var. All which derive from either $prefix -# or $exec_prefix are exported in the form of a Makefile var -# reference, e.g. libdir=${exec_prefix}/lib. Ergo, if +# Each autoconf-relvant --XYZ flag which is explicitly passed to +# configure is exported as-is, as are those which default to some +# top-level system directory, e.g. /etc or /var. All which derive +# from either $prefix or $exec_prefix are exported in the form of a +# Makefile var reference, e.g. libdir=${exec_prefix}/lib. Ergo, if # --exec-prefix=FOO is passed to configure, libdir will still derive, # at make-time, from whatever exec_prefix is passed to make, and will # use FOO if exec_prefix is not overridden at make-time. Without this @@ -1467,7 +1469,7 @@ proc proj-remap-autoconf-dir-vars {} { } # Maintenance reminder: the [join] call is to avoid {braces} # around the output when someone passes in, - # e.g. --libdir=\${prefix}/foo/bar. The Debian package build + # e.g. --libdir=\${prefix}/foo/bar. Debian's SQLite package build # script does that. } } @@ -1526,7 +1528,6 @@ proc proj-current-scope {{lvl 0}} { } } - # # Converts parts of tclConfig.sh to autosetup [define]s. # @@ -1568,7 +1569,7 @@ proc proj-tclConfig-sh-to-autosetup {tclConfigSh} { set fd [open "| sh" "rw"] #puts "fd = $fd"; exit puts $fd $shBody - flush $fd + #flush $fd; # "bad file descriptor"? Without flush, [read] blocks set rd [read $fd] close $fd puts "rd=$rd"; exit 1 @@ -1576,8 +1577,12 @@ proc proj-tclConfig-sh-to-autosetup {tclConfigSh} { } else { set shName ".tclConfigSh.tcl" proj-file-write $shName $shBody - eval [exec sh $shName $tclConfigSh] + catch { + eval [exec sh $shName $tclConfigSh] + expr 1 + } rc xopts file delete -force $shName + return {*}$xopts $rc } } @@ -1635,7 +1640,7 @@ proc proj-tweak-default-env-dirs {} { # If $postProcessScript is not empty then, during # [proj-dot-ins-process], it will be eval'd immediately after # processing the file. In the context of that script, the vars -# $fileIn and $fileOut will be set to the input and output file +# $dotInsIn and $dotInsOut will be set to the input and output file # names. This can be used, for example, to make the output file # executable or perform validation on its contents. # @@ -1713,7 +1718,10 @@ proc proj-dot-ins-process {args} { proj-validate-no-unresolved-ats $fOut } if {"" ne $fScript} { - uplevel 1 "set fileIn $fIn; set fileOut $fOut; eval {$fScript}" + uplevel 1 [join [list set dotInsIn $fIn \; \ + set dotInsOut $fOut \; \ + eval \{${fScript}\} \; \ + unset dotInsIn dotInsOut]] } } if {$flags(-clear)} { diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index 3f7d174d7a..74d5b79317 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -994,7 +994,7 @@ proc sqlite-handle-emsdk {} { proj-bin-define wasm-opt BIN_WASM_OPT } proj-dot-ins-append $emccSh.in $emccSh { - catch {exec chmod u+x $fileOut} + catch {exec chmod u+x $dotInsOut} } proj-dot-ins-append $extWasmConfig.in $extWasmConfig } else { diff --git a/manifest b/manifest index 73a3ec1ae6..0f9f59a7b1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\sfts5matchinfo.test\sfile\sthat\sprevented\sit\sfrom\nworking\son\sMac. -D 2025-04-22T18:45:40.700 +C Upstream\steaish\sfixes\sand\simprovements. +D 2025-04-23T14:54:13.759 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -22,20 +22,20 @@ F autoconf/Makefile.msc 5bc67d3912444c40c6f96d003e5c90663e51abb83d204a520110b1b2 F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac -F autoconf/tea/Makefile.in 88fc65f68618f7ab7d78bded67105d0787e5d71ab183f51d80a9bc5dce2478ab +F autoconf/tea/Makefile.in bb66da485861af82a45e7c135fcf89270f5c0989dbfbb17912c745440b2f9c6c F autoconf/tea/README.txt 656d4686c509d375f5988ff3deda94f65fe6cd8358cd55d1f1dcc7b6e2ff73aa -F autoconf/tea/auto.def 81e2617cfb90d53c19b53b3ec632cd2893bf32f2e5dd272b1116fadf2ea86c2d +F autoconf/tea/auto.def ce95b9450e2fa4ba5dc857e208fe10f4e6f2d737796ac3278aee6079db417529 F autoconf/tea/autosetup/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca -F autoconf/tea/autosetup/core.tcl 9b8415824dff052ce25ce9beed895357d6b7cd8ed1422c76ba9744c3d2f9e7c4 +F autoconf/tea/autosetup/core.tcl b230b6a6d711c162ead7249c771bfb730b7c413b38ceb6832f92baab8d2e62a5 F autoconf/tea/autosetup/feature-tests.tcl 307cac35806817fc87bd0d92004364ee682c0e99f2ab505291500493ab6c7a5f -F autoconf/tea/autosetup/tester.tcl 236598490975a3c071ee753900f3775192b116cf0531c6c754d17e028ea3f880 +F autoconf/tea/autosetup/tester.tcl 93cff35afc5cabc8f855fad114fae58b435fa47cbab5f8cbd4dc694ff0c26107 F autoconf/tea/configure d0b12b984edca6030d1976375b80157ac78b5b90a5b4f0dcee39357f63f4a80b x F autoconf/tea/doc/sqlite3.n 9a97f4f717ceab73004ea412af7960625c1cb24b5c25e4ae4c8b5d8fa4300f4e F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 -F autoconf/tea/pkgIndex.tcl.in 14e8265862e8901a613ef50101f4f813a56f837a63f54e2431b9df0e9bc663a0 -F autoconf/tea/teaish.tcl c8e63cfb1b5cc5f688a90fea16c9bf8b0d6444dce62bd03f4abf7e8cdabf84ce +F autoconf/tea/pkgIndex.tcl.in e07da6b94561f4aa382bab65b1ccceb04701b97bf59d007c1d1f20a222b22d07 +F autoconf/tea/teaish.tcl 9abc575ff3106c60ec0924a558d45dd58f7b4cd07373f87372df1988611b7d8c F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 -F autoconf/tea/teaish.tester.tcl.in 9251c1beb38c24f708a35d74f75a0a1686cc8761034f99d8092282ac10040976 +F autoconf/tea/teaish.tester.tcl.in 430c3514633f6de0a9527f52ff7d251febacf839a91d45c04b10a70388bcaee2 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e F autosetup/README.md f324bb9f9bf1cc787122034df53fbfdfed28ee2657e6652b763d992ab0d04829 @@ -51,8 +51,8 @@ F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e F autosetup/jimsh0.c 563b966c137a4ce3c9333e5196723b7ac0919140a9d7989eb440463cd855c367 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 3eb8e6c74002fc89a1543406039bc36a9347e6deda90fadd00a7436820d38014 -F autosetup/sqlite-config.tcl bde169c42d5d5331485ae7785f5dd19ca1aa01315787e8306441c472ad4e4201 +F autosetup/proj.tcl 38999aca5258e59fc6e7d4d5964c395b74bb5dc7865479bd98cee4dca71e61ba +F autosetup/sqlite-config.tcl 54c88abadda1a42c6276186afcf6534e9bcb08350d96676c4ec08b99f2ada484 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b6c5516940851bd17706435148253325d1b912a44ae138d2b8ca33e5167418c7 -R 9218ccd8b518b209e6a20d61b383e872 -U drh -Z f58d140defc6379d7662d9d02b260125 +P 5a411c609ba0ee17b1ce5bd0b40b9ded92d76f35933f92b2e02a58a3f169c715 +R b0c338fe00382bfb412fdd47cce333a1 +U stephan +Z eee13aa4761e9cdaf3c2c73939710886 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 357f794a79..3bf8bc9ec0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a411c609ba0ee17b1ce5bd0b40b9ded92d76f35933f92b2e02a58a3f169c715 +235ea2c0e1823fe4324c3890069a9131ebb6de14e55eb56c0e9f42a7e491a699 From ef90c4209cefa1391c7217aee5978a53b3b8152b Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 25 Apr 2025 11:27:13 +0000 Subject: [PATCH 08/42] Teaish internal API tweaks and ensure that -encoding utf-8 is used when source'ing test scripts. FossilOrigin-Name: f878f578f8fac95a939e3eabe8b64d5627d2ccd98de1e28081b9bca42ffee027 --- autoconf/tea/autosetup/core.tcl | 53 +++++++++++++++++-------------- autoconf/tea/teaish.tester.tcl.in | 6 ++-- autosetup/proj.tcl | 4 +-- manifest | 16 +++++----- manifest.uuid | 2 +- 5 files changed, 43 insertions(+), 38 deletions(-) diff --git a/autoconf/tea/autosetup/core.tcl b/autoconf/tea/autosetup/core.tcl index 9ef0178e29..660173b328 100644 --- a/autoconf/tea/autosetup/core.tcl +++ b/autoconf/tea/autosetup/core.tcl @@ -177,8 +177,8 @@ proc teaish-configure-core {} { # TEA has --with-tclinclude but it appears to only be useful for # building an extension against an uninstalled copy of TCL's own - # source tree. Either we get that info from tclConfig.sh or we - # give up. + # source tree. The policy here is that either we get that info + # from tclConfig.sh or we give up. # # with-tclinclude:DIR # => {Specify the directory which contains the tcl.h. This should not @@ -188,24 +188,27 @@ proc teaish-configure-core {} { # extensions, and thus use a teaish-... prefix on most flags. However, # --teaish-extension-dir is frequently needed, so... # - # As of this spontaneous moment, we'll formalize using using - # --t-X-Y to abbreviate teaish flags when doing so is + # As of this spontaneous moment, we'll settle on using --t-A-X to + # abbreviate --teaish-A...-X... flags when doing so is # unambiguous... ted: t-e-d: teaish-extension-dir:DIR - => {Looks for an extension in the given directory instead of the current dir.} + => {Looks for an extension in the given directory instead of the current + dir.} t-c-e: teaish-create-extension:TARGET_DIRECTORY => {Writes stub files for creating an extension. Will refuse to overwrite - existing files without --force.} + existing files without --teaish-force.} t-f teaish-force - => {Has a context-dependent meaning (autosetup defines --force for its own use)} + => {Has a context-dependent meaning (autosetup defines --force for its + own use).} t-d-d - teaish-dump-defines => {Dump all configure-defined vars to config.defines.txt} + teaish-dump-defines + => {Dump all configure-defined vars to config.defines.txt} t-v teaish-verbose=0 @@ -813,7 +816,7 @@ proc teaish__find_extension {} { if {$dirExt ne $dirSrc} { lappend flist $dirSrc/teaish.tcl } - if {![proj-first-file-found $flist extT]} { + if {![proj-first-file-found extT $flist]} { if {$gotHelpArg} { # Tell teaish-configure-core that the lack of extension is not # an error when --help is used. @@ -859,9 +862,8 @@ If you are attempting an out-of-tree build, use # We use the first one of teaish.make.in or teaish.make we find in # $dirExt. # - if {[proj-first-file-found \ - [list $dirExt/teaish.make.in $dirExt/teaish.make] \ - extM]} { + if {[proj-first-file-found extM \ + [list $dirExt/teaish.make.in $dirExt/teaish.make]]} { if {[string match *.in $extM]} { define TEAISH_MAKEFILE_IN $extM define TEAISH_MAKEFILE [file rootname [file tail $extM]] @@ -878,9 +880,8 @@ If you are attempting an out-of-tree build, use } # Look for teaish.pkginit.tcl[.in] - if {[proj-first-file-found \ - [list $dirExt/teaish.pkginit.tcl.in $dirExt/teaish.pkginit.tcl] \ - extI]} { + if {[proj-first-file-found extI \ + [list $dirExt/teaish.pkginit.tcl.in $dirExt/teaish.pkginit.tcl]]} { if {[string match *.in $extI]} { proj-dot-ins-append $extI define TEAISH_PKGINIT_TCL_IN $extI @@ -897,19 +898,19 @@ If you are attempting an out-of-tree build, use # Look for pkgIndex.tcl[.in]... set piPolicy 0 - if {[proj-first-file-found $dirExt/pkgIndex.tcl.in extPI]} { + if {[proj-first-file-found extPI $dirExt/pkgIndex.tcl.in]} { # Generate ./pkgIndex.tcl from it. define TEAISH_PKGINDEX_TCL_IN $extPI define TEAISH_PKGINDEX_TCL [file rootname [file tail $extPI]] apply $addDist $extPI set piPolicy 0x01 } elseif {$dirExt ne $dirSrc - && [proj-first-file-found $dirSrc/pkgIndex.tcl.in extPI]} { + && [proj-first-file-found extPI $dirSrc/pkgIndex.tcl.in]} { # Generate ./pkgIndex.tcl from it. define TEAISH_PKGINDEX_TCL_IN $extPI define TEAISH_PKGINDEX_TCL [file rootname [file tail $extPI]] set piPolicy 0x02 - } elseif {[proj-first-file-found $dirExt/pkgIndex.tcl extPI]} { + } elseif {[proj-first-file-found extPI $dirExt/pkgIndex.tcl]} { # Assume it's a static file and use it. define TEAISH_PKGINDEX_TCL_IN "" define TEAISH_PKGINDEX_TCL $extPI @@ -924,7 +925,7 @@ If you are attempting an out-of-tree build, use # Look for teaish.test.tcl[.in] proj-assert {"" ne $dirExt} set flist [list $dirExt/teaish.test.tcl.in $dirExt/teaish.test.tcl] - if {[proj-first-file-found $flist ttt]} { + if {[proj-first-file-found ttt $flist]} { if {[string match *.in $ttt]} { # Generate teaish.test.tcl from $ttt set xt [file rootname [file tail $ttt]] @@ -944,7 +945,7 @@ If you are attempting an out-of-tree build, use # Look for teaish.tester.tcl[.in] set flist [list $dirExt/teaish.tester.tcl.in $dirSrc/teaish.tester.tcl.in] - if {[proj-first-file-found $flist ttt]} { + if {[proj-first-file-found ttt $flist]} { # Generate teaish.test.tcl from $ttt set xt [file rootname [file tail $ttt]] file delete -force -- $xt; # ensure no stale copy is used @@ -1408,10 +1409,14 @@ proc teaish__pragma {args} { # teaish-pkginfo-set ?-vars|-subst? {-name foo -version 0.1.2} # # The latter may be easier to write for a multi-line invocation. -# Passing the -vars flag tells it to perform a [subst] of (only) -# variables in the {...} part from the calling scope. The -subst flag -# will cause is to [subst] the {...} with command substitution as well -# (but no backslash substitution). +# +# For the second call form, passing the -vars flag tells it to perform +# a [subst] of (only) variables in the {...} part from the calling +# scope. The -subst flag will cause it to [subst] the {...} with +# command substitution as well (but no backslash substitution). When +# using -subst for string concatenation, e.g. with -libDir +# foo[get-version-number], be sure to wrap the value in braces: +# -libDir {foo[get-version-number]}. # # Each pkginfo flag corresponds to one piece of extension package # info. Teaish provides usable default values for all of these flags, diff --git a/autoconf/tea/teaish.tester.tcl.in b/autoconf/tea/teaish.tester.tcl.in index 8f1362a75c..802f1d9754 100644 --- a/autoconf/tea/teaish.tester.tcl.in +++ b/autoconf/tea/teaish.tester.tcl.in @@ -19,11 +19,11 @@ if {[llength [lindex $::argv 0]] > 0} { load [lindex $::argv 0] [lindex $::argv 1]; } -source [lindex $::argv 2]; # teaish/tester.tcl +source -encoding utf-8 [lindex $::argv 2]; # teaish/tester.tcl @if TEAISH_PKGINIT_TCL apply {{file} { set dir [file dirname $::argv0] - source $file + source -encoding utf-8 $file }} [join {@TEAISH_PKGINIT_TCL@}] @endif @if TEAISH_TEST_TCL @@ -32,7 +32,7 @@ apply {{file} { array set ::teaish__BuildFlags @TEAISH__DEFINES_MAP@ set dir [file normalize [file dirname $file]] #test-fail "Just testing" - source $file + source -encoding utf-8 $file }} [join {@TEAISH_TEST_TCL@}] @else # TEAISH_TEST_TCL # No $TEAISH_TEST_TCL provided, so here's a default test which simply diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 1ccc632809..3ebb5f2500 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -1760,13 +1760,13 @@ proc proj-validate-no-unresolved-ats {args} { } # -# @proj-first-found fileList tgtVar +# @proj-first-found tgtVar fileList # # Searches $fileList for an existing file. If one is found, its name is # assigned to tgtVar and 1 is returned, else tgtVar is not modified # and 0 is returned. # -proc proj-first-file-found {fileList tgtVar} { +proc proj-first-file-found {tgtVar fileList} { upvar $tgtVar tgt foreach f $fileList { if {[file exists $f]} { diff --git a/manifest b/manifest index 0f9f59a7b1..50f9d69f8e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Upstream\steaish\sfixes\sand\simprovements. -D 2025-04-23T14:54:13.759 +C Teaish\sinternal\sAPI\stweaks\sand\sensure\sthat\s-encoding\sutf-8\sis\sused\swhen\ssource'ing\stest\sscripts. +D 2025-04-25T11:27:13.409 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -26,7 +26,7 @@ F autoconf/tea/Makefile.in bb66da485861af82a45e7c135fcf89270f5c0989dbfbb17912c74 F autoconf/tea/README.txt 656d4686c509d375f5988ff3deda94f65fe6cd8358cd55d1f1dcc7b6e2ff73aa F autoconf/tea/auto.def ce95b9450e2fa4ba5dc857e208fe10f4e6f2d737796ac3278aee6079db417529 F autoconf/tea/autosetup/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca -F autoconf/tea/autosetup/core.tcl b230b6a6d711c162ead7249c771bfb730b7c413b38ceb6832f92baab8d2e62a5 +F autoconf/tea/autosetup/core.tcl 603506c76d1bdf3cf12be0ed87e11fb77aeae3b0598b97e3989e6fabb60de01a F autoconf/tea/autosetup/feature-tests.tcl 307cac35806817fc87bd0d92004364ee682c0e99f2ab505291500493ab6c7a5f F autoconf/tea/autosetup/tester.tcl 93cff35afc5cabc8f855fad114fae58b435fa47cbab5f8cbd4dc694ff0c26107 F autoconf/tea/configure d0b12b984edca6030d1976375b80157ac78b5b90a5b4f0dcee39357f63f4a80b x @@ -35,7 +35,7 @@ F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in e07da6b94561f4aa382bab65b1ccceb04701b97bf59d007c1d1f20a222b22d07 F autoconf/tea/teaish.tcl 9abc575ff3106c60ec0924a558d45dd58f7b4cd07373f87372df1988611b7d8c F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 -F autoconf/tea/teaish.tester.tcl.in 430c3514633f6de0a9527f52ff7d251febacf839a91d45c04b10a70388bcaee2 +F autoconf/tea/teaish.tester.tcl.in 0d048e5569ad9bbaffbe5123c2084f9084d424a7719d06f5ca941caafbcca320 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e F autosetup/README.md f324bb9f9bf1cc787122034df53fbfdfed28ee2657e6652b763d992ab0d04829 @@ -51,7 +51,7 @@ F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e F autosetup/jimsh0.c 563b966c137a4ce3c9333e5196723b7ac0919140a9d7989eb440463cd855c367 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 38999aca5258e59fc6e7d4d5964c395b74bb5dc7865479bd98cee4dca71e61ba +F autosetup/proj.tcl 7eff53437b4e0a5cc9215221487129ea8c53b89e8b651ee2d2713cdc6e1d0642 F autosetup/sqlite-config.tcl 54c88abadda1a42c6276186afcf6534e9bcb08350d96676c4ec08b99f2ada484 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5a411c609ba0ee17b1ce5bd0b40b9ded92d76f35933f92b2e02a58a3f169c715 -R b0c338fe00382bfb412fdd47cce333a1 +P 235ea2c0e1823fe4324c3890069a9131ebb6de14e55eb56c0e9f42a7e491a699 +R ea525321199f0f66dd9876a7d0687939 U stephan -Z eee13aa4761e9cdaf3c2c73939710886 +Z 5ae1dc01157b80ae73825ec4ac34cc82 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3bf8bc9ec0..634d3c9f7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -235ea2c0e1823fe4324c3890069a9131ebb6de14e55eb56c0e9f42a7e491a699 +f878f578f8fac95a939e3eabe8b64d5627d2ccd98de1e28081b9bca42ffee027 From 0f0450eda34c306886078f580eb92fdb779d1f2e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 25 Apr 2025 12:39:32 +0000 Subject: [PATCH 09/42] Fix an off-by-one error in an assert(), discovered by oss-fuzz. This is a harmless error in as much as assert()s are disabled in production builds, and because the off-by-one only occurs on nonsensical CREATE INDEX statements. FossilOrigin-Name: 3e627d66ebdef8dfe41bd12a0474d1afca9f50051771774679d25bd1833e69ae --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 50f9d69f8e..abd86ba1ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Teaish\sinternal\sAPI\stweaks\sand\sensure\sthat\s-encoding\sutf-8\sis\sused\swhen\ssource'ing\stest\sscripts. -D 2025-04-25T11:27:13.409 +C Fix\san\soff-by-one\serror\sin\san\sassert(),\sdiscovered\sby\soss-fuzz.\s\sThis\sis\sa\nharmless\serror\sin\sas\smuch\sas\sassert()s\sare\sdisabled\sin\sproduction\sbuilds,\sand\nbecause\sthe\soff-by-one\sonly\soccurs\son\snonsensical\sCREATE\sINDEX\sstatements. +D 2025-04-25T12:39:32.313 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -729,7 +729,7 @@ F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea F src/btree.c 00fcee37947641f48d4b529d96143e74d056b7afa8f26d61292c90ee59c056b2 F src/btree.h 18e5e7b2124c23426a283523e5f31a4bff029131b795bb82391f9d2f3136fc50 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886 -F src/build.c ad72c60b6e01053dea9d1e03f52de747138e4b0de1c73f17501ea307cf40a240 +F src/build.c 67c1db4c5e89a8519fe9b6dafc287f6bc3627696b5b8536dc5e06db570d8c05f F src/callback.c acae8c8dddda41ee85cfdf19b926eefe830f371069f8aadca3aa39adf5b1c859 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9 @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 235ea2c0e1823fe4324c3890069a9131ebb6de14e55eb56c0e9f42a7e491a699 -R ea525321199f0f66dd9876a7d0687939 -U stephan -Z 5ae1dc01157b80ae73825ec4ac34cc82 +P f878f578f8fac95a939e3eabe8b64d5627d2ccd98de1e28081b9bca42ffee027 +R b09672114488a7538b124c2f208fd0c9 +U drh +Z ec15e368363710db096ef47cd437c3e8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 634d3c9f7d..ca8e4b50d1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f878f578f8fac95a939e3eabe8b64d5627d2ccd98de1e28081b9bca42ffee027 +3e627d66ebdef8dfe41bd12a0474d1afca9f50051771774679d25bd1833e69ae diff --git a/src/build.c b/src/build.c index 13f5b71330..5bd3aac3ca 100644 --- a/src/build.c +++ b/src/build.c @@ -1071,7 +1071,7 @@ int sqlite3TableColumnToIndex(Index *pIdx, int iCol){ int i; i16 iCol16; assert( iCol>=(-1) && iCol<=SQLITE_MAX_COLUMN ); - assert( pIdx->nColumn<=SQLITE_MAX_COLUMN ); + assert( pIdx->nColumn<=SQLITE_MAX_COLUMN+1 ); iCol16 = iCol; for(i=0; inColumn; i++){ if( iCol16==pIdx->aiColumn[i] ){ From c49c7dad89ddbf39668abf4bf650ef434347346a Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 27 Apr 2025 04:21:27 +0000 Subject: [PATCH 10/42] autoconf/tea: the default value of --threadsafe=X is now based on a pkgconfig query using the target tclsh, which is much more reliable than grepping Tcl's linker flags. Doc touchups. FossilOrigin-Name: 4aeec30443b282f10353cdb9415fdce436287280f0f5d5d6b9917da831744898 --- autoconf/tea/teaish.tcl | 60 ++++++++++++++++++++--------------------- manifest | 14 +++++----- manifest.uuid | 2 +- 3 files changed, 37 insertions(+), 39 deletions(-) diff --git a/autoconf/tea/teaish.tcl b/autoconf/tea/teaish.tcl index 53228cb910..99db68e249 100644 --- a/autoconf/tea/teaish.tcl +++ b/autoconf/tea/teaish.tcl @@ -1,4 +1,4 @@ -# Teaish configure script for the SQLite TCL extension +# Teaish configure script for the SQLite Tcl extension apply {{} { set version [proj-file-content -trim [teaish-get -dir]/../VERSION] @@ -31,7 +31,9 @@ array set sqliteConfig [subst [proj-strip-hash-comments { # autosetup's [options] function. # proc teaish-options {} { - return [proj-strip-hash-comments [subst -nocommands -nobackslashes { + # These flags and defaults mostly derive from the historical TEA + # build. Some, like ICU, are taken from the canonical SQLite tree. + return [subst -nocommands -nobackslashes { with-system-sqlite=0 => {Use the system-level SQLite instead of the copy in this tree. Also requires use of --override-sqlite-version so that the build @@ -63,7 +65,7 @@ proc teaish-options {} { icu-collations=0 => {Enable SQLITE_ENABLE_ICU_COLLATIONS. Requires --with-icu-ldflags=... or --with-icu-config} - }]] + }] } # @@ -118,9 +120,8 @@ proc teaish-configure {} { sqlite-handle-common-feature-flags; # must be late in the process }; # teaish-configure - define OPT_FEATURE_FLAGS {} ; # -DSQLITE_OMIT/ENABLE flags. -######################################################################## +# # Adds $args, if not empty, to OPT_FEATURE_FLAGS. This is intended only for holding # -DSQLITE_ENABLE/OMIT/... flags, but that is not enforced here. proc sqlite-add-feature-flag {args} { @@ -129,7 +130,7 @@ proc sqlite-add-feature-flag {args} { } } -######################################################################## +# # Check for log(3) in libm and die with an error if it is not # found. $featureName should be the feature name which requires that # function (it's used only in error messages). defines LDFLAGS_MATH to @@ -150,7 +151,7 @@ proc sqlite-affirm-have-math {featureName} { } } -######################################################################## +# # Handle various SQLITE_ENABLE/OMIT_... feature flags. proc sqlite-handle-common-feature-flags {} { msg-result "Feature flags..." @@ -208,7 +209,7 @@ proc sqlite-handle-common-feature-flags {} { } } } - ######################################################################## + # # Invert the above loop's logic for some SQLITE_OMIT_... cases. If # config option $boolFlag is false, [sqlite-add-feature-flag # $featureFlag], where $featureFlag is intended to be @@ -224,7 +225,7 @@ proc sqlite-handle-common-feature-flags {} { } } - ######################################################################### + ## # Remove duplicates from the final feature flag sets and show them # to the user. set oFF [get-define OPT_FEATURE_FLAGS] @@ -238,7 +239,7 @@ proc sqlite-handle-common-feature-flags {} { teaish-cflags-add -define OPT_FEATURE_FLAGS }; # sqlite-handle-common-feature-flags -######################################################################## +# # If --enable-threadsafe is set, this adds -DSQLITE_THREADSAFE=1 to # OPT_FEATURE_FLAGS and sets LDFLAGS_PTHREAD to the linker flags # needed for linking pthread (possibly an empty string). If @@ -260,10 +261,8 @@ proc sqlite-handle-threadsafe {} { teaish-ldflags-prepend $ldf undefine lib_pthread_create undefine lib_pthread_mutexattr_init - } elseif {[proj-opt-was-provided threadsafe]} { - user-error "Missing required pthread libraries. Use --disable-threadsafe to disable this check." } else { - msg-result "pthread support not detected" + user-error "Missing required pthread libraries. Use --disable-threadsafe to disable this check." } # Recall that LDFLAGS_PTHREAD might be empty even if pthreads if # found because it's in -lc on some platforms. @@ -273,30 +272,31 @@ proc sqlite-handle-threadsafe {} { } else { # # If user does not specify --[disable-]threadsafe then select a - # default based on whether it looks like TCL has threading + # default based on whether it looks like Tcl has threading # support. # - #puts "TCL_LIBS = [get-define TCL_LIBS]" - if {[string match *pthread* [get-define TCL_LIBS]]} { - # ^^^ FIXME: there must be a better way of testing this + catch { + scan [exec echo {puts [tcl::pkgconfig get threaded]} | [get-define TCLSH_CMD]] \ + %d enable + } + if {$enable} { set flagName "--threadsafe" set lblAbled "enabled" - set enable 1 - msg-result "yes" + msg-result yes } else { set flagName "--disable-threadsafe" set lblAbled "disabled" - set enable 0 - msg-result "no" + msg-result no } - msg-result "NOTICE: defaulting to ${flagName} because TCL has threading ${lblAbled}." - # ^^^ We don't need to link against -lpthread in the is-enabled case. + msg-result "Defaulting to ${flagName} because Tcl has threading ${lblAbled}." + # ^^^ We (probably) don't need to link against -lpthread in the + # is-enabled case. We might in the case of static linking. Unsure. } sqlite-add-feature-flag -DSQLITE_THREADSAFE=${enable} return $enable } -######################################################################## +# # Handles the --enable-load-extension flag. Returns 1 if the support # is enabled, else 0. If support for that feature is not found, a # fatal error is triggered if --enable-load-extension is explicitly @@ -348,7 +348,7 @@ proc sqlite-handle-load-extension {} { return $found } -######################################################################## +# # ICU - International Components for Unicode # # Handles these flags: @@ -448,7 +448,7 @@ proc sqlite-handle-icu {} { }; # sqlite-handle-icu -######################################################################## +# # Handles the --with-tempstore flag. # # The test fixture likes to set SQLITE_TEMP_STORE on its own, so do @@ -473,7 +473,7 @@ proc sqlite-handle-tempstore {} { } } -######################################################################## +# # Handles the --enable-math flag. proc sqlite-handle-math {} { proj-if-opt-truthy math { @@ -492,15 +492,13 @@ proc sqlite-handle-math {} { } } -######################################################################## +# # Move -DSQLITE_OMIT... and -DSQLITE_ENABLE... flags from CFLAGS and # CPPFLAGS to OPT_FEATURE_FLAGS and remove them from BUILD_CFLAGS. proc sqlite-munge-cflags {} { # Move CFLAGS and CPPFLAGS entries matching -DSQLITE_OMIT* and # -DSQLITE_ENABLE* to OPT_FEATURE_FLAGS. This behavior is derived - # from the legacy build and was missing the 3.48.0 release (the - # initial Autosetup port). - # https://sqlite.org/forum/forumpost/9801e54665afd728 + # from the pre-3.48 build. # # Handling of CPPFLAGS, as well as removing ENABLE/OMIT from # CFLAGS/CPPFLAGS, was missing in the 3.49.0 release as well. diff --git a/manifest b/manifest index abd86ba1ef..ecf49a35dc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\serror\sin\san\sassert(),\sdiscovered\sby\soss-fuzz.\s\sThis\sis\sa\nharmless\serror\sin\sas\smuch\sas\sassert()s\sare\sdisabled\sin\sproduction\sbuilds,\sand\nbecause\sthe\soff-by-one\sonly\soccurs\son\snonsensical\sCREATE\sINDEX\sstatements. -D 2025-04-25T12:39:32.313 +C autoconf/tea:\sthe\sdefault\svalue\sof\s--threadsafe=X\sis\snow\sbased\son\sa\spkgconfig\squery\susing\sthe\starget\stclsh,\swhich\sis\smuch\smore\sreliable\sthan\sgrepping\sTcl's\slinker\sflags.\sDoc\stouchups. +D 2025-04-27T04:21:27.214 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -33,7 +33,7 @@ F autoconf/tea/configure d0b12b984edca6030d1976375b80157ac78b5b90a5b4f0dcee39357 F autoconf/tea/doc/sqlite3.n 9a97f4f717ceab73004ea412af7960625c1cb24b5c25e4ae4c8b5d8fa4300f4e F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in e07da6b94561f4aa382bab65b1ccceb04701b97bf59d007c1d1f20a222b22d07 -F autoconf/tea/teaish.tcl 9abc575ff3106c60ec0924a558d45dd58f7b4cd07373f87372df1988611b7d8c +F autoconf/tea/teaish.tcl 172fd9d6e06e5874db58e3384cfa9a7d52325634302eed189bb60f6cbf40bbd9 F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 F autoconf/tea/teaish.tester.tcl.in 0d048e5569ad9bbaffbe5123c2084f9084d424a7719d06f5ca941caafbcca320 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f878f578f8fac95a939e3eabe8b64d5627d2ccd98de1e28081b9bca42ffee027 -R b09672114488a7538b124c2f208fd0c9 -U drh -Z ec15e368363710db096ef47cd437c3e8 +P 3e627d66ebdef8dfe41bd12a0474d1afca9f50051771774679d25bd1833e69ae +R 56906ec6bb47507a448b4eb61a665ed8 +U stephan +Z 6249b8349b5cb161fd945b7e77195de1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ca8e4b50d1..ee89ce3655 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e627d66ebdef8dfe41bd12a0474d1afca9f50051771774679d25bd1833e69ae +4aeec30443b282f10353cdb9415fdce436287280f0f5d5d6b9917da831744898 From c26cde3bfdfb7df5d8230d7426e8505f7edd166e Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 27 Apr 2025 06:01:26 +0000 Subject: [PATCH 11/42] proj.tcl: correct the auto-reconfigure rules to include any X=Y passed to configure. Improve handling of quoted CFLAGS. FossilOrigin-Name: 7a2f4f319c90b768c6acc96671a3173c0ef0493180f3c4d5443f1ca03d92f6b4 --- autosetup/proj.tcl | 91 +++++++++++++++++++++++----------------------- manifest | 12 +++--- manifest.uuid | 2 +- 3 files changed, 52 insertions(+), 53 deletions(-) diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 3ebb5f2500..bdc593cd43 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -1760,7 +1760,7 @@ proc proj-validate-no-unresolved-ats {args} { } # -# @proj-first-found tgtVar fileList +# @proj-first-file-found tgtVar fileList # # Searches $fileList for an existing file. If one is found, its name is # assigned to tgtVar and 1 is returned, else tgtVar is not modified @@ -1783,19 +1783,10 @@ proc proj-first-file-found {tgtVar fileList} { # can be used to automatically reconfigure. # proc proj-setup-autoreconfig {defName} { - set squote {{arg} { - # Wrap $arg in single-quotes if it looks like it might need that - # to avoid mis-handling as a shell argument. We assume that $arg - # will never contain any single-quote characters. - if {[string match {*[ &;$*"]*} $arg]} { return '$arg' } - return $arg - }} - define-append $defName cd [apply $squote $::autosetup(builddir)] \ - && [apply $squote $::autosetup(srcdir)/configure] - #{*}$::autosetup(argv) breaks with --flag='val with spaces', so... - foreach arg $::autosetup(argv) { - define-append $defName [apply $squote $arg] - } + define $defName \ + [join [list \ + cd \"$::autosetup(builddir)\" \ + && [get-define AUTOREMAKE "error - missing @AUTOREMAKE@"]]] } # @@ -1813,21 +1804,22 @@ proc proj-define-append {defineName args} { } # -# @prod-define-amend ?-p|-prepend? ?-define? FLAG args... +# @prod-define-amend ?-p|-prepend? ?-d|-define? defineName args... # # A proxy for Autosetup's [define-append]. # -# Appends all non-empty $args to the define named by $FLAG unless. If +# Appends all non-empty $args to the define named by $defineName. If # one of (-p | -prepend) are used it instead prepends them, in their -# given order, to $FLAG. +# given order, to $defineName. # # If -define is used then each argument is assumed to be a [define]'d # flag and [get-define X ""] is used to fetch it. # -# Typically, -lXYZ flags need to be in "reverse" order, with each -lY -# resolving symbols for -lX's to its left. This order is largely -# historical, and not relevant on all environments, but it is -# technically correct and still relevant on some environments. +# Re. linker flags: typically, -lXYZ flags need to be in "reverse" +# order, with each -lY resolving symbols for -lX's to its left. This +# order is largely historical, and not relevant on all environments, +# but it is technically correct and still relevant on some +# environments. # # See: proj-append-to # @@ -1838,9 +1830,9 @@ proc proj-define-amend {args} { set xargs [list] foreach arg $args { switch -exact -- $arg { - -p - -prepend { set prepend 1 } - -d - -define { set isdefs 1 } "" {} + -p - -prepend { incr prepend } + -d - -define { incr isdefs } default { if {"" eq $defName} { set defName $arg @@ -1850,6 +1842,9 @@ proc proj-define-amend {args} { } } } + if {"" eq $defName} { + proj-error "Missing defineName argument in call from [proj-current-scope 1]" + } if {$isdefs} { set args $xargs set xargs [list] @@ -1871,47 +1866,51 @@ proc proj-define-amend {args} { } # -# @proj-define-to-cflag ?-list? defineName... +# @proj-define-to-cflag ?-list? ?-quote? ?-zero-undef? defineName... # -# Treat each argument as the name of a [define] -# and attempt to render it like a CFLAGS value: +# Treat each argument as the name of a [define] and renders it like a +# CFLAGS value in one of the following forms: # # -D$name -# -D$name=value +# -D$name=integer (strict integer matches only) +# '-D$name=value' (without -quote) +# '-D$name="value"' (with -quote) # -# If treats integers as numbers and everything else as a quoted +# It treats integers as numbers and everything else as a quoted # string, noting that it does not handle strings which themselves # contain quotes. # +# The -zero-undef flag causes no -D to be emitted for integer values +# of 0. +# # By default it returns the result as string of all -D... flags, # but if passed the -list flag it will return a list of the # individual CFLAGS. # proc proj-define-to-cflag {args} { set rv {} - set xargs {} - set returnList 0; - foreach arg $args { - switch -exact -- $arg { - -list {incr returnList} - default { - lappend xargs $arg - } - } + proj-parse-simple-flags args flags { + -list 0 {expr 1} + -quote 0 {expr 1} + -zero-undef 0 {expr 1} } - foreach d $xargs { + foreach d $args { set v [get-define $d ""] - set li [list -D${d}] - if {[string is integer -strict $v]} { - lappend li = $v - } elseif {"" eq $d} { + set li {} + if {"" eq $d} { + set v "-D${d}" + } elseif {[string is integer -strict $v]} { + if {!$flags(-zero-undef) || $v ne "0"} { + set v "-D${d}=$v" + } + } elseif {$flags(-quote)} { + set v "'-D${d}=\"$v\"'" } else { - lappend li = {"} $v {"} + set v "'-D${d}=$v'" } - lappend rv [join $li ""] + lappend rv $v } - if {$returnList} { return $rv } - return [join $rv] + expr {$flags(-list) ? $rv : [join $rv]} } diff --git a/manifest b/manifest index ecf49a35dc..15e4fa0d6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C autoconf/tea:\sthe\sdefault\svalue\sof\s--threadsafe=X\sis\snow\sbased\son\sa\spkgconfig\squery\susing\sthe\starget\stclsh,\swhich\sis\smuch\smore\sreliable\sthan\sgrepping\sTcl's\slinker\sflags.\sDoc\stouchups. -D 2025-04-27T04:21:27.214 +C proj.tcl:\scorrect\sthe\sauto-reconfigure\srules\sto\sinclude\sany\sX=Y\spassed\sto\sconfigure.\sImprove\shandling\sof\squoted\sCFLAGS. +D 2025-04-27T06:01:26.900 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -51,7 +51,7 @@ F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e F autosetup/jimsh0.c 563b966c137a4ce3c9333e5196723b7ac0919140a9d7989eb440463cd855c367 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 7eff53437b4e0a5cc9215221487129ea8c53b89e8b651ee2d2713cdc6e1d0642 +F autosetup/proj.tcl d2c0c6da22b4b6e0da6d054851079477815f8acc265e718f16879f3e88ffd819 F autosetup/sqlite-config.tcl 54c88abadda1a42c6276186afcf6534e9bcb08350d96676c4ec08b99f2ada484 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3e627d66ebdef8dfe41bd12a0474d1afca9f50051771774679d25bd1833e69ae -R 56906ec6bb47507a448b4eb61a665ed8 +P 4aeec30443b282f10353cdb9415fdce436287280f0f5d5d6b9917da831744898 +R d75d037808a96903ea777bfc48335881 U stephan -Z 6249b8349b5cb161fd945b7e77195de1 +Z 7e9da962d1e2273ff3dad52e3e71aeeb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ee89ce3655..6428295eb6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4aeec30443b282f10353cdb9415fdce436287280f0f5d5d6b9917da831744898 +7a2f4f319c90b768c6acc96671a3173c0ef0493180f3c4d5443f1ca03d92f6b4 From 886f674198514be07731b66009abc575450dda6e Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 28 Apr 2025 09:25:19 +0000 Subject: [PATCH 12/42] tea: pull in upstream teaish. Consolidate the build definition from the upstream build and this one to ease maintenace. Improve compatibility with historical TEA usage of --with-tcl=... (it insists on a /lib suffix for that path). FossilOrigin-Name: 0ea5993dd8549056d73f3d5447e0a810495ece23a75be8add78fba6842125d85 --- autoconf/tea/Makefile.in | 194 ++++-- autoconf/tea/autosetup/core.tcl | 988 +++++++++++++++++++++++------- autoconf/tea/autosetup/tester.tcl | 9 +- autoconf/tea/teaish.tcl | 95 ++- autosetup/proj.tcl | 7 +- manifest | 20 +- manifest.uuid | 2 +- 7 files changed, 969 insertions(+), 346 deletions(-) diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index 60c8761d62..5aa374cf78 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -34,8 +34,8 @@ INSTALL.noexec = $(INSTALL) -m 0644 # any given teaish build. # tx.name = @TEAISH_NAME@ -tx.pkgName = @TEAISH_PKGNAME@ tx.version = @TEAISH_VERSION@ +tx.name.pkg = @TEAISH_PKGNAME@ tx.libdir = @TEAISH_LIBDIR_NAME@ tx.loadPrefix = @TEAISH_LOAD_PREFIX@ tx.tcl = @TEAISH_TCL@ @@ -48,6 +48,12 @@ tx.dll9 = @TEAISH_DLL9@ tx.dll = $(tx.dll$(TCL_MAJOR_VERSION)) tx.dir = @TEAISH_EXT_DIR@ +@if TEAISH_DIST_NAME +tx.name.dist = @TEAISH_DIST_NAME@ +@else +tx.name.dist = $(teaish.name) +@endif + teaish.dir = @abs_top_srcdir@ #teaish.dir.autosetup = @TEAISH_AUTOSETUP_DIR@ teaish.makefile = Makefile @@ -69,6 +75,7 @@ libdir = @libdir@ libexecdir = @libexecdir@ localstatedir = @localstatedir@ mandir = @mandir@ +prefix = @prefix@ runstatedir = @runstatedir@ sbindir = @sbindir@ sharedstatedir = @sharedstatedir@ @@ -81,16 +88,17 @@ sysconfdir = @sysconfdir@ # TCLSH = @TCLSH_CMD@ TCL_CONFIG_SH = @TCL_CONFIG_SH@ -TCL_INCLUDE_SPEC = @TCL_INCLUDE_SPEC@ -TCL_LIB_SPEC = @TCL_LIB_SPEC@ -TCL_STUB_LIB_SPEC = @TCL_STUB_LIB_SPEC@ TCL_EXEC_PREFIX = @TCL_EXEC_PREFIX@ -TCL_VERSION = @TCL_VERSION@ +TCL_INCLUDE_SPEC = @TCL_INCLUDE_SPEC@ +TCL_LIBS = @TCL_LIBS@ +TCL_LIB_SPEC = @TCL_LIB_SPEC@ TCL_MAJOR_VERSION = @TCL_MAJOR_VERSION@ TCL_MINOR_VERSION = @TCL_MINOR_VERSION@ TCL_PATCH_LEVEL = @TCL_PATCH_LEVEL@ +TCL_PREFIX = @TCL_PREFIX@ TCL_SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@ -TCL_LIBS = @TCL_LIBS@ +TCL_STUB_LIB_SPEC = @TCL_STUB_LIB_SPEC@ +TCL_VERSION = @TCL_VERSION@ TCLLIBDIR = @TCLLIBDIR@ # @@ -107,10 +115,15 @@ CFLAGS.configure = @SH_CFLAGS@ @TEAISH_CFLAGS@ @CFLAGS@ @CPPFLAGS@ $(TCL_INCLUDE # LDFLAGS.configure = LDFLAGS as known at configure-time. # # This ordering is deliberate: flags populated via tcl's -# [teaish-ldflags-add] should preceed LDFLAGS (which typically +# [teaish-ldflags-add] should precede LDFLAGS (which typically # comes from the ./configure command-line invocation). # -LDFLAGS.configure = @SH_LDFLAGS@ @TEAISH_LDFLAGS@ @LDFLAGS@ $(TCL_STUB_LIB_SPEC) +LDFLAGS.configure = @TEAISH_LDFLAGS@ @LDFLAGS@ + +# +# Linker flags for linkhing a shared library. +# +LDFLAGS.shlib = @SH_LDFLAGS@ # # The following tx.XYZ vars may be populated/modified by teaish.tcl @@ -119,9 +132,9 @@ LDFLAGS.configure = @SH_LDFLAGS@ @TEAISH_LDFLAGS@ @LDFLAGS@ $(TCL_STUB_LIB_SPEC) # # tx.src is the list of source or object files to include in the -# (single) compiler invocation. This will initially contain any -# sources passed to [teaish-src-add], but may also be appended to -# by teaish.make. +# (single) compiler/linker invocation. This will initially contain any +# sources passed to [teaish-src-add], but may also be appended to by +# teaish.make. # tx.src =@TEAISH_EXT_SRC@ @@ -163,25 +176,49 @@ teaish__autogen.deps = \ @TEAISH_PKGINDEX_TCL_IN@ \ @AUTODEPS@ +@if TEAISH_MAKEFILE_IN +$(tx.makefile): $(tx.makefile.in) +@endif + +teaish.autoreconfig = \ + @TEAISH_AUTORECONFIG@ + # # Problem: when more than one target can invoke TEAISH_AUTORECONFIG, # we can get parallel reconfigures running. Thus, targets which # may require reconfigure should depend on... # config.log: $(teaish__autogen.deps) - @TEAISH_AUTORECONFIG@ + $(teaish.autoreconfig) # ^^^ We would love to skip this when running [dist]clean, but there's # no POSIX Make-portable way to do that. GNU Make can. .PHONY: reconfigure reconfigure: - @TEAISH_AUTORECONFIG@ + $(teaish.autoreconfig) $(teaish.makefile): $(teaish__auto.def) $(teaish.makefile.in) \ @AUTODEPS@ +@if TEAISH_TESTER_TCL_IN @TEAISH_TESTER_TCL_IN@: @TEAISH_TESTER_TCL@: @TEAISH_TESTER_TCL_IN@ config.log: @TEAISH_TESTER_TCL@ +@endif + +# +# CC variant for compiling Tcl-using sources. +# +CC.tcl = \ + $(CC) -o $@ $(CFLAGS.configure) $(CFLAGS) $(tx.CFLAGS) + +# +# CC variant for linking $(tx.src) into an extension DLL. Note that +# $(tx.src) must come before $(LDFLAGS...) for linking to third-party +# libs to work. +# +CC.dll = \ + $(CC.tcl) $(tx.src) $(LDFLAGS.shlib) \ + $(LDFLAGS.configure) $(LDFLAGS) $(tx.LDFLAGS) $(TCL_STUB_LIB_SPEC) @if TEAISH_ENABLE_DLL # @@ -189,8 +226,11 @@ config.log: @TEAISH_TESTER_TCL@ # target: the extension shared lib. # $(tx.dll): $(tx.src) config.log - $(CC) -o $@ $(CFLAGS.configure) $(CFLAGS) $(tx.CFLAGS) \ - $(tx.src) $(LDFLAGS.configure) $(LDFLAGS) $(tx.LDFLAGS) + @if [ "x" = "x$(tx.src)" ]; then \ + echo "Makefile var tx.src (source/object files) is empty" 1>&2; \ + exit 1; \ + fi + $(CC.dll) all: $(tx.dll) @endif # TEAISH_ENABLE_DLL @@ -208,6 +248,7 @@ test-extension: # this name is reserved for use by teaish.make[.in] @if TEAISH_ENABLE_DLL test-prepre: $(tx.dll) @endif +@if TEAISH_TESTER_TCL test-core.args = @TEAISH_TESTER_TCL@ @if TEAISH_ENABLE_DLL test-core.args += '$(tx.dll)' '$(tx.loadPrefix)' @@ -215,10 +256,14 @@ test-core.args += '$(tx.dll)' '$(tx.loadPrefix)' test-core.args += '' '' @endif test-core.args += @TEAISH_TESTUTIL_TCL@ -test-prepre: @TEAISH_TESTER_TCL@ -test-pre: test-prepre test-core: test-pre $(TCLSH) $(test-core.args) +test-prepre: @TEAISH_TESTER_TCL@ +@else # !TEAISH_TESTER_TCL +test-prepre: +@endif # TEAISH_TESTER_TCL +test-pre: test-prepre +test-core: test-pre test-post: test-core test: test-post @@ -234,7 +279,6 @@ clean-post: clean-core clean: clean-post .PHONY: distclean-pre distclean-core distclean-post clean-extension -distclean-extension: # this name is reserved for use by teaish.make distclean-pre: clean distclean-core: distclean-pre rm -f Makefile @@ -256,6 +300,7 @@ distclean-core: distclean-pre @if TEAISH_TEST_TCL_IN rm -f @TEAISH_TEST_TCL@ @endif +distclean-extension: # this name is reserved for use by teaish.make distclean-post: distclean-core distclean: distclean-post @@ -277,14 +322,15 @@ install-core: install-pre $(INSTALL.noexec) @TEAISH_PKGINIT_TCL@ "$(DESTDIR)$(TCLLIBDIR)" @endif install-test: install-core - @echo 'package require $(tx.pkgName) $(tx.version)' > $@.tcl - @echo "Post-install test of [package require $(tx.pkgName) $(tx.version)]..." - @if $(TCLSH) $@.tcl ; then \ - echo "test passed"; \ - rm -f $@.tcl; \ + @echo "Post-install test of [package require $(tx.name.pkg) $(tx.version)]..."; \ + if echo \ + 'set c 0;' \ + 'if {[catch {package require $(tx.name.pkg) $(tx.version)}]} {incr c};' \ + 'exit $$c' \ + | $(TCLSH) ; then \ + echo "passed"; \ else \ - echo "TEST FAILED"; \ - rm -f $@.tcl; \ + echo "FAILED"; \ exit 1; \ fi install-post: install-test @@ -302,35 +348,50 @@ uninstall-post: uninstall-core @echo "Uninstalled Tcl extension $(tx.name) $(tx.version)" uninstall: uninstall-post -Makefile: config.log $(teaish.makefile.in) @if TEAISH_MAKEFILE_IN -config.log: @TEAISH_MAKEFILE_IN@ +Makefile: $(tx.makefile.in) +config.log: $(teaish.makefile.in) @endif # # Package archive generation ("dist") rules... # @if TEAISH_ENABLE_DIST +@if BIN_TAR @if BIN_ZIP + +# When installing teaish as part of "make dist", we need to run +# configure with similar flags to what we last configured with but we +# must not pass on any extension-specific flags, as those won't be +# recognized when running in --teaish-install mode, causing +# the sub-configure to fail. +dist.flags = --with-tclsh=$(TCLSH) +dist.reconfig = $(teaish.dir)/configure $(dist.flags) + # Temp dir for dist.zip. Must be different than dist.tgz or else # parallel builds may hose the dist. teaish__dist.tmp.zip = teaish__dist_zip # # Make a distribution zip file... # -dist.zip.dir = $(tx.name)-$(tx.version) -dist.zip = $(dist.zip.dir).zip +dist.basename = $(tx.name.dist)-$(tx.version) +dist.zip = $(dist.basename).zip .PHONY: dist.zip dist.zip-core dist.zip-post #dist.zip-pre: # We apparently can't add a pre-hook here, else "make dist" rebuilds # the archive each time it's run. $(dist.zip): $(tx.dist.files) @rm -fr $(teaish__dist.tmp.zip) - @mkdir -p $(teaish__dist.tmp.zip)/$(dist.zip.dir) + @mkdir -p $(teaish__dist.tmp.zip)/$(dist.basename) @tar cf $(teaish__dist.tmp.zip)/tmp.tar $(tx.dist.files) - @tar xf $(teaish__dist.tmp.zip)/tmp.tar -C $(teaish__dist.tmp.zip)/$(dist.zip.dir) - @rm -f $(dist.zip.dir)/tmp.tar $(dist.zip) - @cd $(teaish__dist.tmp.zip) && zip -q -r ../$(dist.zip) $(dist.zip.dir) + @tar xf $(teaish__dist.tmp.zip)/tmp.tar -C $(teaish__dist.tmp.zip)/$(dist.basename) +@if TEAISH_DIST_FULL + @$(dist.reconfig) \ + --teaish-install=$(teaish__dist.tmp.zip)/$(dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(dist.basename) >/dev/null +@endif + @rm -f $(dist.basename)/tmp.tar $(dist.zip) + @cd $(teaish__dist.tmp.zip) && zip -q -r ../$(dist.zip) $(dist.basename) @rm -fr $(teaish__dist.tmp.zip) @ls -la $(dist.zip) dist.zip-core: $(dist.zip) @@ -342,23 +403,27 @@ undist-zip: undist: undist-zip @endif #BIN_ZIP -@if BIN_TAR # # Make a distribution tarball... # teaish__dist.tmp.tgz = teaish__dist_tgz -dist.tgz.dir = $(tx.name)-$(tx.version) -dist.tgz = $(dist.tgz.dir).tar.gz +dist.tgz = $(dist.basename).tar.gz .PHONY: dist.tgz dist.tgz-core dist.tgz-post # dist.tgz-pre: # see notes in dist.zip $(dist.tgz): $(tx.dist.files) @rm -fr $(teaish__dist.tmp.tgz) - @mkdir -p $(teaish__dist.tmp.tgz)/$(dist.tgz.dir) + @mkdir -p $(teaish__dist.tmp.tgz)/$(dist.basename) @tar cf $(teaish__dist.tmp.tgz)/tmp.tar $(tx.dist.files) - @tar xf $(teaish__dist.tmp.tgz)/tmp.tar -C $(teaish__dist.tmp.tgz)/$(dist.tgz.dir) - @rm -f $(dist.tgz.dir)/tmp.tar $(dist.tgz) - @cd $(teaish__dist.tmp.tgz) && tar czf ../$(dist.tgz) $(dist.tgz.dir) + @tar xf $(teaish__dist.tmp.tgz)/tmp.tar -C $(teaish__dist.tmp.tgz)/$(dist.basename) +@if TEAISH_DIST_FULL + @rm -f $(teaish__dist.tmp.tgz)/$(dist.basename)/pkgIndex.tcl.in; # kludge + @$(dist.reconfig) \ + --teaish-install=$(teaish__dist.tmp.tgz)/$(dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(dist.basename) >/dev/null +@endif + @rm -f $(dist.basename)/tmp.tar $(dist.tgz) + @cd $(teaish__dist.tmp.tgz) && tar czf ../$(dist.tgz) $(dist.basename) @rm -fr $(teaish__dist.tmp.tgz) @ls -la $(dist.tgz) dist.tgz-core: $(dist.tgz) @@ -368,6 +433,9 @@ dist: dist.tgz undist-tgz: rm -f $(dist.tgz) undist: undist-tgz +@else #!BIN_TAR +dist: + @echo "The dist rules require tar, which configure did not find." 1>&2; exit 1 @endif #BIN_TAR @else #!TEAISH_ENABLE_DIST undist: @@ -378,6 +446,21 @@ dist: @endif @endif #TEAISH_ENABLE_DIST +Makefile: @TEAISH_TCL@ + +@if TEAISH_MAKEFILE_CODE +# +# TEAISH_MAKEFILE_CODE may contain literal makefile code, which +# gets pasted verbatim here. Either [define TEAISH_MAKEFILE_CODE +# ...] or use [teaish-make-add] to incrementally build up this +# content. +# +# +@TEAISH_MAKEFILE_CODE@ +# +@endif + +@if TEAISH_MAKEFILE # # TEAISH_MAKEFILE[_IN] defines any extension-specific state this file # needs. @@ -401,31 +484,12 @@ dist: # It may optionally hook into various targets as documented in # /doc/extensions.md in the canonical teaish source tree. # -# Interestingly, we don't have to pre-filter teaish.makefile.in - -# we can just import it into here. That skips its teaish-specific -# validation though. Hmm. +# Interestingly, we don't have to pre-filter teaish.makefile.in - we +# can just @include it here. That skips its teaish-specific validation +# though. Hmm. # -#@if TEAISH_MAKEFILE_IN -## TEAISH_MAKEFILE_IN ==> -#Makefile: @TEAISH_MAKEFILE_IN@ -#@include @TEAISH_MAKEFILE_IN@ -#@endif -#@if !TEAISH_MAKEFILE_IN -@if TEAISH_MAKEFILE -# TEAISH_MAKEFILE ==> +# Makefile: @TEAISH_MAKEFILE@ @include @TEAISH_MAKEFILE@ -@endif #TEAISH_MAKEFILE -#@endif #!TEAISH_MAKEFILE_IN - -# -# TEAISH_MAKEFILE_CODE may contain literal makefile code, which -# gets pasted verbatim here. Either [define TEAISH_MAKEFILE_CODE -# ...] or use [teaish-make-add] to incrementally build up this -# content. -# -@if TEAISH_MAKEFILE_CODE -# TEAISH_MAKEFILE_CODE ==> -Makefile: @TEAISH_TCL@ -@TEAISH_MAKEFILE_CODE@ -@endif #TEAISH_MAKEFILE_CODE +# +@endif diff --git a/autoconf/tea/autosetup/core.tcl b/autoconf/tea/autosetup/core.tcl index 660173b328..c0baeb8fe5 100644 --- a/autoconf/tea/autosetup/core.tcl +++ b/autoconf/tea/autosetup/core.tcl @@ -34,6 +34,7 @@ array set teaish__Config [proj-strip-hash-comments { # set to 1 to enable some internal debugging output debug-enabled 0 + # # 0 = don't yet have extension's pkgindex # 0x01 = found TEAISH_EXT_DIR/pkgIndex.tcl.in @@ -46,6 +47,20 @@ array set teaish__Config [proj-strip-hash-comments { # pkgindex-policy 0 + # + # The pkginit counterpart of pkgindex-policy: + # + # 0 = no pkginit + # 0x01 = found default X.in: generate X from X.in + # 0x10 = found static pkginit file X + # 0x02 = user-provided X.in generates ./X. + # 0x20 = user-provided static pkginit file X + # + # The 0x0f bits indicate that teaish is responsible for cleaning up + # the (generated) pkginit file. + # + pkginit-policy 0 + # # If 1+ then teaish__verbose will emit messages. # @@ -53,16 +68,21 @@ array set teaish__Config [proj-strip-hash-comments { # # Mapping of pkginfo -flags to their TEAISH_xxx define (if any). + # This must not be modified. # pkginfo-f2d { - -name TEAISH_NAME - -pkgName TEAISH_PKGNAME - -version TEAISH_VERSION - -libDir TEAISH_LIBDIR_NAME - -loadPrefix TEAISH_LOAD_PREFIX - -vsatisfies TEAISH_VSATISFIES - -options {} - -pragmas {} + -name TEAISH_NAME + -name.dist TEAISH_DIST_NAME + -name.pkg TEAISH_PKGNAME + -version TEAISH_VERSION + -libDir TEAISH_LIBDIR_NAME + -loadPrefix TEAISH_LOAD_PREFIX + -vsatisfies TEAISH_VSATISFIES + -pkgInit.tcl TEAISH_PKGINIT_TCL + -pkgInit.tcl.in TEAISH_PKGINIT_TCL_IN + -url TEAISH_URL + -options {} + -pragmas {} } # @@ -93,6 +113,15 @@ array set teaish__Config [proj-strip-hash-comments { # Dir where teaish.tcl is found. extension-dir {} + + # Whether the generates TEASH_VSATISFIES_CODE should error out on a + # satisfies error. If 0, it uses return instead of error. + vsatisfies-error 1 + + # Whether or not to allow a "full dist" - a "make dist" build which + # includes both the extension and teaish. By default this is only on + # if the extension dir is teaish's dir. + dist-full-enabled 0 }] set teaish__Config(core-dir) $::autosetup(libdir)/teaish @@ -139,8 +168,8 @@ if {[teaish-argv-has --teaish-verbose --t-v]} { msg-quiet use system ; # Outputs "Host System" and "Build System" lines if {"--help" ni $::argv} { teaish__verbose 1 msg-result "TEA(ish) Version = $::teaish__Config(version)" - teaish__verbose 1 msg-result "Source dir = $::autosetup(srcdir)" - teaish__verbose 1 msg-result "Build dir = $::autosetup(builddir)" + teaish__verbose 1 msg-result "Source dir = $::autosetup(srcdir)" + teaish__verbose 1 msg-result "Build dir = $::autosetup(builddir)" } # @@ -152,9 +181,13 @@ if {"--help" ni $::argv} { proc teaish-configure-core {} { proj-tweak-default-env-dirs + set ::teaish__Config(install-mode) [teaish-argv-has --teaish-install*] + set ::teaish__Config(create-ext-mode) \ + [teaish-argv-has --teaish-create-extension=* --t-c-e=*] set gotExt 0; # True if an extension config is found - if {![teaish-argv-has --teaish-create-extension=* --t-c-e=*]} { - # Don't look for an extension if we're in --t-c-e mode + if {!$::teaish__Config(create-ext-mode) + && !$::teaish__Config(install-mode)} { + # Don't look for an extension if we're in --t-c-e or --t-i mode set gotExt [teaish__find_extension] } @@ -210,22 +243,43 @@ proc teaish-configure-core {} { teaish-dump-defines => {Dump all configure-defined vars to config.defines.txt} - t-v - teaish-verbose=0 + t-v:=0 + teaish-verbose:=0 => {Enable more (often extraneous) messages from the teaish core.} t-d - teaish-debug => {Enable teaish-specific debug output} + teaish-debug=0 => {Enable teaish-specific debug output} + + t-i + teaish-install:=auto + => {Installs a copy of teaish, including autosetup, to the target dir. + When used with --teaish-create-extension=DIR, a value of "auto" + (no no value) will inherit that directory.} + + #TODO: --teaish-install-extension:=dir as short for + # --t-c-e=dir --t-i + + t-e-p: + teaish-extension-pkginfo:pkginfo + => {For use with --teaish-create-extension. If used, it must be a + list of arguments for use with teaish-pkginfo-set, e.g. + --teaish-extension-pkginfo="-name Foo -version 2.3"} + + t-v-c + teaish-vsatisfies-check=1 + => {Disable the configure-time "vsatisfies" check on the target tclsh.} + }]; # main options. if {$gotExt} { proj-assert {"" ne [teaish-pkginfo-get -name]} proj-assert {[file exists $::teaish__Config(teaish.tcl)]} \ "Expecting to have found teaish.tcl by now" - uplevel 1 [list source $::teaish__Config(teaish.tcl)] + uplevel 1 {source $::teaish__Config(teaish.tcl)} # Set up some default values if the extension did not set them. # This must happen _after_ it's sourced but before # teaish-configure is called. + teaish__f2d_array f2d foreach {pflag key type val} { - TEAISH_CFLAGS -v "" - TEAISH_LDFLAGS -v "" @@ -234,29 +288,38 @@ proc teaish-configure-core {} { - TEAISH_MAKEFILE_IN -v "" - TEAISH_PKGINDEX_TCL -v "" - TEAISH_PKGINDEX_TCL_IN -v "" - - TEAISH_PKGINIT_TCL -v "" - - TEAISH_PKGINIT_TCL_IN -v "" - TEAISH_TEST_TCL -v "" - TEAISH_TEST_TCL_IN -v "" - -version TEAISH_VERSION -v 0.0.0 - -pkgName TEAISH_PKGNAME -e {teaish-pkginfo-get -name} - -libDir TEAISH_LIBDIR_NAME -e { + -version :f2d: -v 0.0.0 + -name.pkg :f2d: -e {teaish-pkginfo-get -name} + -name.dist :f2d: -e {teaish-pkginfo-get -name} + -libDir :f2d: -e { join [list \ - [teaish-pkginfo-get -pkgName] \ + [teaish-pkginfo-get -name.pkg] \ [teaish-pkginfo-get -version]] "" } - -loadPrefix TEAISH_LOAD_PREFIX -e { - string totitle [teaish-get -pkgName] + -loadPrefix :f2d: -e { + string totitle [teaish-get -name.pkg] } - -vsatisfies TEAISH_VSATISFIES -v {{Tcl 8.5-}} + -vsatisfies :f2d: -v {{Tcl 8.5-}} + -pkgInit.tcl :f2d: -v "" + -pkgInit.tcl.in :f2d: -v "" + -url :f2d: -v "" } { - set isDefOnly [expr {"-" eq $pflag}] - if {!$isDefOnly && [info exists ::teaish__PkgInfo($pflag)]} { - continue; + set isPIFlag [expr {"-" ne $pflag}] + if {$isPIFlag} { + if {[info exists ::teaish__PkgInfo($pflag)]} { + # Was already set - skip it. + continue; + } + proj-assert {{:f2d:} eq $key} + set key $f2d($pflag) } + proj-assert {"" ne $key} set got [get-define $key ""] - if {$isDefOnly && "" ne $got} { + if {"" ne $got} { + # Was already set - skip it. continue } switch -exact -- $type { @@ -264,13 +327,14 @@ proc teaish-configure-core {} { -e { set val [eval $val] } default { proj-error "Invalid type flag: $type" } } - #puts "***** defining default $pflag $key {$val} isDefOnly=$isDefOnly got=$got" + #puts "***** defining default $pflag $key {$val} isPIFlag=$isPIFlag got=$got" define $key $val - if {!$isDefOnly} { + if {$isPIFlag} { set ::teaish__PkgInfo($pflag) $val } } - unset isDefOnly pflag key type val + unset isPIFlag pflag key type val + array unset f2d }; # sourcing extension's teaish.tcl if {[llength [info proc teaish-options]] > 0} { @@ -297,15 +361,28 @@ proc teaish-configure-core {} { t-d-d => teaish-dump-defines ted => teaish-extension-dir t-e-d => teaish-extension-dir + t-e-p => teaish-extension-pkginfo t-f => teaish-force + t-i => teaish-install t-v => teaish-verbose + t-v-c => teaish-vsatisfies-check } - set ::teaish__Config(verbose) [opt-bool teaish-verbose] + scan [opt-val teaish-verbose 0] %d ::teaish__Config(verbose) set ::teaish__Config(debug-enabled) [opt-bool teaish-debug] + set exitEarly 0 if {[proj-opt-was-provided teaish-create-extension]} { teaish__create_extension [opt-val teaish-create-extension] + incr exitEarly + } + if {$::teaish__Config(install-mode)} { + teaish__install + incr exitEarly + } + + if {$exitEarly} { + file delete -force config.log return } proj-assert {1==$gotExt} "Else we cannot have gotten this far" @@ -377,24 +454,11 @@ proc teaish__configure_phase1 {} { } }}; # --[exec-]prefix defaults teaish__check_common_bins - # # Set up library file names # proj-file-extensions - apply {{} { - set name [teaish-pkginfo-get -name]; # _not_ -pkgName - set pkgver [teaish-pkginfo-get -version] - set libname "lib" - if {[string match *-cygwin [get-define host]]} { - set libname cyg - } - define TEAISH_DLL8_BASENAME $libname$name$pkgver - define TEAISH_DLL9_BASENAME ${libname}tcl9$name$pkgver - set ext [get-define TARGET_DLLEXT] - define TEAISH_DLL8 [get-define TEAISH_DLL8_BASENAME]$ext - define TEAISH_DLL9 [get-define TEAISH_DLL9_BASENAME]$ext - }} + teaish__define_pkginfo_derived * teaish-checks-run -pre if {[llength [info proc teaish-configure]] > 0} { @@ -404,25 +468,50 @@ proc teaish__configure_phase1 {} { } teaish-checks-run -post - if {0} { - # Reminder: we cannot do a TEAISH_VSATISFIES_TCL check like the - # following from here because _this_ tcl instance is very possibly - # not the one which will be hosting the extension. This part might - # be running in JimTcl, which can't load the being-configured - # extension. - if {$::autosetup(istcl)} { - # ^^^ this is a canonical Tcl, not JimTcl - set vsat [get-define TEAISH_VSATISFIES_TCL ""] - if {$vsat ne "" - && ![package vsatisfies [package provide Tcl] $vsat]} { - error [join [list "Tcl package vsatisfies failed for" \ - [teaish-pkginfo-get -name] \ - [teaish-pkginfo-get -version] \ - ": expecting vsatisfies to match ($vsat)"]] + apply {{} { + # Set up "vsatisfies" code for pkgIndex.tcl.in, + # teaish.tester.tcl.in, and for a configure-time check. We would + # like to put this before [teaish-checks-run -pre] but it's + # marginally conceivable that a client may need to dynamically + # calculate the vsatisfies and set it via [teaish-configure]. + set vs [get-define TEAISH_VSATISFIES ""] + if {"" eq $vs} return + set code {} + set n 0 + # Treat $vs as a list-of-lists {{Tcl 8.5-} {Foo 1.0- -3.0} ...} + # and generate Tcl which will run package vsatisfies tests with + # that info. + foreach pv $vs { + set n [llength $pv] + if {$n < 2} { + proj-error "-vsatisfies: {$pv} appears malformed. Whole list is: $vs" } - unset vsat - } + set pkg [lindex $pv 0] + set vcheck {} + for {set i 1} {$i < $n} {incr i} { + lappend vcheck [lindex $pv $i] + } + if {[opt-bool teaish-vsatisfies-check]} { + set tclsh [get-define TCLSH_CMD] + set vsat "package vsatisfies \[ package provide $pkg \] $vcheck" + set vputs "puts \[ $vsat \]" + #puts "*** vputs = $vputs" + scan [exec echo $vputs | $tclsh] %d vvcheck + if {0 == $vvcheck} { + proj-fatal -up $tclsh "check failed:" $vsat + } + } + lappend code [string trim [subst -nocommands -nobackslashes { +if { ![package vsatisfies [package provide $pkg] $vcheck] } { + if {$::teaish__Config(vsatisfies-error)} { + error {Package $::teaish__PkgInfo(-name) $::teaish__PkgInfo(-version) requires $pv} + } else { + return } +}}]] + }; # foreach pv + define TEAISH_VSATISFIES_CODE [join $code "\n"] + }}; # vsatisfies if {[proj-looks-like-windows]} { # Without this, linking of an extension will not work on Cygwin or @@ -458,43 +547,44 @@ proc teaish__configure_phase1 {} { } }}; # $::teaish__Config(pkgindex-policy) - set dEx $::teaish__Config(extension-dir) - set dSrc $::autosetup(srcdir) - - proj-dot-ins-append $dSrc/Makefile.in - proj-dot-ins-append $dSrc/teaish.tester.tcl.in + # + # Ensure we clean up TEAISH_PKGINIT_TCL if needed and @-process + # TEAISH_PKGINIT_TCL_IN if needed. + # + if {0x0f & $::teaish__Config(pkginit-policy)} { + file delete -force -- [get-define TEAISH_PKGINIT_TCL] + proj-dot-ins-append [get-define TEAISH_PKGINIT_TCL_IN] + } apply {{} { - set vs [get-define TEAISH_VSATISFIES ""] - if {"" eq $vs} return - # Treat as a list-of-lists {{Tcl 8.5-} {Foo 1.0- -3.0} ...} and - # generate Tcl which will run package vsatisfies tests with that - # info. - set code {} - set n 0 - foreach pv $vs { - set n [llength $pv] - if {$n < 2} { - proj-error "-vsatisfies: {$pv} appears malformed. Whole list is: $vs" - } - set ifpv [list if \{ "!\[package vsatisfies \[package provide"] - lappend ifpv [lindex $pv 0] "\]" - for {set i 1} {$i < $n} {incr i} { - lappend ifpv [lindex $pv $i] - } - lappend ifpv "\] \} \{\n" - lappend ifpv \ - "error \{Package $::teaish__PkgInfo(-name) $::teaish__PkgInfo(-version) requires $pv\}\n" \ - "\}" - lappend code [join $ifpv] + # Queue up any remaining dot-in files + set dotIns [list] + foreach d { + TEAISH_TESTER_TCL_IN + TEAISH_TEST_TCL_IN + TEAISH_MAKEFILE_IN + } { + lappend dotIns [get-define $d ""] + } + lappend dotIns $::autosetup(srcdir)/Makefile.in; # must be after TEAISH_MAKEFILE_IN + foreach f $dotIns { + if {"" ne $f} { + proj-dot-ins-append $f + } } - define TEAISH_VSATISFIES_CODE [join $code "\n"] }} + define TEAISH_DIST_FULL \ + [expr { + $::teaish__Config(dist-enabled) + && $::teaish__Config(dist-full-enabled) + }] + + define TEAISH_AUTOSETUP_DIR $::teaish__Config(core-dir) define TEAISH_ENABLE_DIST $::teaish__Config(dist-enabled) define TEAISH_ENABLE_DLL $::teaish__Config(dll-enabled) - define TEAISH_AUTOSETUP_DIR $::teaish__Config(core-dir) define TEAISH_TCL $::teaish__Config(teaish.tcl) + define TEAISH_DIST_FILES [join $::teaish__Config(dist-files)] define TEAISH_EXT_DIR [join $::teaish__Config(extension-dir)] define TEAISH_EXT_SRC [join $::teaish__Config(extension-src)] @@ -533,7 +623,7 @@ proc teaish__configure_phase1 {} { # the configure process. # #proj-file-write $::autosetup(builddir)/.configured "" -} +}; # teaish__configure_phase1 # # Run checks for required binaries. @@ -581,65 +671,72 @@ proc teaish__check_tcl {} { set srcdir $::autosetup(srcdir) msg-result "Checking for a suitable tcl... " set use_tcl 1 - set with_tclsh [opt-val with-tclsh [proj-get-env TCLSH]] - set with_tcl [opt-val with-tcl [proj-get-env TCL_HOME]] + set withSh [opt-val with-tclsh [proj-get-env TCLSH]] + set tclHome [opt-val with-tcl [proj-get-env TCL_HOME]] + if {[string match */lib $tclHome]} { + # TEA compatibility kludge: its --with-tcl wants the lib + # dir containing tclConfig.sh. + #proj-warn "Replacing --with-tcl=$tclHome for TEA compatibility" + regsub {/lib^} $tclHome "" tclHome + msg-result "NOTE: stripped /lib suffix from --with-tcl=$tclHome (a TEA-ism)" + } if {0} { # This misinteracts with the $TCL_PREFIX default: it will use the # autosetup-defined --prefix default - if {"prefix" eq $with_tcl} { - set with_tcl [get-define prefix] + if {"prefix" eq $tclHome} { + set tclHome [get-define prefix] } } teaish-debug "use_tcl ${use_tcl}" - teaish-debug "with_tclsh=${with_tclsh}" - teaish-debug "with_tcl=$with_tcl" - if {"" eq $with_tclsh && "" eq $with_tcl} { + teaish-debug "withSh=${withSh}" + teaish-debug "tclHome=$tclHome" + if {"" eq $withSh && "" eq $tclHome} { # If neither --with-tclsh nor --with-tcl are provided, try to find # a workable tclsh. - set with_tclsh [proj-first-bin-of tclsh9.1 tclsh9.0 tclsh8.6 tclsh] - teaish-debug "with_tclsh=${with_tclsh}" + set withSh [proj-first-bin-of tclsh9.1 tclsh9.0 tclsh8.6 tclsh] + teaish-debug "withSh=${withSh}" } set doConfigLookup 1 ; # set to 0 to test the tclConfig.sh-not-found cases - if {"" ne $with_tclsh} { + if {"" ne $withSh} { # --with-tclsh was provided or found above. Validate it and use it # to trump any value passed via --with-tcl=DIR. - if {![file-isexec $with_tclsh]} { - proj-error "TCL shell $with_tclsh is not executable" + if {![file-isexec $withSh]} { + proj-error "TCL shell $withSh is not executable" } else { - define TCLSH_CMD $with_tclsh - #msg-result "Using tclsh: $with_tclsh" + define TCLSH_CMD $withSh + #msg-result "Using tclsh: $withSh" } if {$doConfigLookup && - [catch {exec $with_tclsh $::autosetup(libdir)/find_tclconfig.tcl} result] == 0} { - set with_tcl $result + [catch {exec $withSh $::autosetup(libdir)/find_tclconfig.tcl} result] == 0} { + set tclHome $result } - if {"" ne $with_tcl && [file isdirectory $with_tcl]} { - teaish__verbose 1 msg-result "$with_tclsh recommends the tclConfig.sh from $with_tcl" + if {"" ne $tclHome && [file isdirectory $tclHome]} { + teaish__verbose 1 msg-result "$withSh recommends the tclConfig.sh from $tclHome" } else { - proj-warn "$with_tclsh is unable to recommend a tclConfig.sh" + proj-warn "$withSh is unable to recommend a tclConfig.sh" set use_tcl 0 } } set cfg "" - set tclSubdirs {tcl9.1 tcl9.0 tcl8.6 lib} + set tclSubdirs {tcl9.1 tcl9.0 tcl8.6 tcl8.5 lib} while {$use_tcl} { - if {"" ne $with_tcl} { - # Ensure that we can find tclConfig.sh under ${with_tcl}/... + if {"" ne $tclHome} { + # Ensure that we can find tclConfig.sh under ${tclHome}/... if {$doConfigLookup} { - if {[file readable "${with_tcl}/tclConfig.sh"]} { - set cfg "${with_tcl}/tclConfig.sh" + if {[file readable "${tclHome}/tclConfig.sh"]} { + set cfg "${tclHome}/tclConfig.sh" } else { foreach i $tclSubdirs { - if {[file readable "${with_tcl}/$i/tclConfig.sh"]} { - set cfg "${with_tcl}/$i/tclConfig.sh" + if {[file readable "${tclHome}/$i/tclConfig.sh"]} { + set cfg "${tclHome}/$i/tclConfig.sh" break } } } } if {"" eq $cfg} { - proj-error "No tclConfig.sh found under ${with_tcl}" + proj-error "No tclConfig.sh found under ${tclHome}" } } else { # If we have not yet found a tclConfig.sh file, look in $libdir @@ -663,14 +760,14 @@ proc teaish__check_tcl {} { } teaish__verbose 1 msg-result "Using tclConfig.sh = $cfg" break - } + }; # while {$use_tcl} define TCL_CONFIG_SH $cfg # Export a subset of tclConfig.sh to the current TCL-space. If $cfg # is an empty string, this emits empty-string entries for the # various options we're interested in. proj-tclConfig-sh-to-autosetup $cfg - if {"" eq $with_tclsh && $cfg ne ""} { + if {"" eq $withSh && $cfg ne ""} { # We have tclConfig.sh but no tclsh. Attempt to locate a tclsh # based on info from tclConfig.sh. set tclExecPrefix [get-define TCL_EXEC_PREFIX] @@ -680,23 +777,23 @@ proc teaish__check_tcl {} { $tclExecPrefix/bin/tclsh ] foreach trySh $tryThese { if {[file-isexec $trySh]} { - set with_tclsh $trySh + set withSh $trySh break } } - if {![file-isexec $with_tclsh]} { + if {![file-isexec $withSh]} { proj-warn "Cannot find a usable tclsh (tried: $tryThese)" } } - define TCLSH_CMD $with_tclsh + define TCLSH_CMD $withSh if {$use_tcl} { # Set up the TCLLIBDIR set tcllibdir [get-env TCLLIBDIR ""] set extDirName [get-define TEAISH_LIBDIR_NAME] if {"" eq $tcllibdir} { # Attempt to extract TCLLIBDIR from TCL's $auto_path - if {"" ne $with_tclsh && - [catch {exec echo "puts stdout \$auto_path" | "$with_tclsh"} result] == 0} { + if {"" ne $withSh && + [catch {exec echo "puts stdout \$auto_path" | "$withSh"} result] == 0} { foreach i $result { if {[file isdirectory $i]} { set tcllibdir $i/$extDirName @@ -710,8 +807,8 @@ proc teaish__check_tcl {} { define TCLLIBDIR $tcllibdir }; # find TCLLIBDIR - if {[file-isexec $with_tclsh]} { - teaish__verbose 1 msg-result "Using tclsh = $with_tclsh" + if {[file-isexec $withSh]} { + teaish__verbose 1 msg-result "Using tclsh = $withSh" if {$cfg ne ""} { define HAVE_TCL 1 } else { @@ -723,17 +820,17 @@ proc teaish__check_tcl {} { # fatally, else just emit a warning. If we can find the APIs needed # to generate a working JimTCL then that will suffice for build-time # TCL purposes (see: proc sqlite-determine-codegen-tcl). - if {![file-isexec $with_tclsh]} { + if {![file-isexec $withSh]} { proj-error "Did not find tclsh" } elseif {"" eq $cfg} { proj-indented-notice -error { Cannot find a usable tclConfig.sh file. Use - --with-tcl=DIR to specify a directory where tclConfig.sh can be + --with-tcl=DIR to specify a directory near which tclConfig.sh can be found, or --with-tclsh=/path/to/tclsh to allow the tclsh binary to locate its tclConfig.sh. } } - msg-result "Using Tcl [get-define TCL_VERSION]." + msg-result "Using Tcl [get-define TCL_VERSION] from [get-define TCL_PREFIX]." }; # teaish__check_tcl # @@ -743,16 +840,16 @@ proc teaish__check_tcl {} { # not find an extension but the --help flag was seen, in which case # that's not an error. # -# This does not _load_ the extension, it simply locates the files -# which make up an extension. -# -# This sets up lots of defines, e.g. TEAISH_EXT_DIR. +# This does not _load_ the extension, it primarily locates the files +# which make up an extension and fills out no small amount of teaish +# state related to that. # proc teaish__find_extension {} { - + proj-assert {!$::teaish__Config(install-mode)} teaish__verbose 1 msg-result "Looking for teaish extension..." + # Helper for the foreach loop below. - set lambdaMT {{mustHave fid dir} { + set checkTeaishTcl {{mustHave fid dir} { if {[file isdirectory $dir]} { set f [file join $dir $fid] if {[file readable $f]} { @@ -765,12 +862,12 @@ proc teaish__find_extension {} { } return "" }} + # # We have to handle some flags manually because the extension must # be loaded before [options] is run (so that the extension can # inject its own options). # - #set extM ""; # teaish.make.in set dirBld $::autosetup(builddir); # dir we're configuring under set dirSrc $::autosetup(srcdir); # where teaish's configure script lives set extT ""; # teaish.tcl @@ -789,11 +886,12 @@ proc teaish__find_extension {} { if {![file isdirectory $extD]} { proj-error "--teaish-extension-dir value is not a directory: $extD" } - set extT [apply $lambdaMT 1 teaish.tcl $extD] + set extT [apply $checkTeaishTcl 1 teaish.tcl $extD] set ::teaish__Config(extension-dir) $extD } --help { incr gotHelpArg + lappend largv $arg } default { lappend largv $arg @@ -802,9 +900,7 @@ proc teaish__find_extension {} { } set ::argv $largv - set dirExt [proj-coalesce \ - $::teaish__Config(extension-dir) \ - $dirBld]; # dir with the extension + set dirExt $::teaish__Config(extension-dir); # dir with the extension # # teaish.tcl is a TCL script which implements various # interfaces described by this framework. @@ -812,14 +908,13 @@ proc teaish__find_extension {} { # We use the first one we find in the builddir or srcdir. # if {"" eq $extT} { - set flist [list $dirExt/teaish.tcl] - if {$dirExt ne $dirSrc} { - lappend flist $dirSrc/teaish.tcl - } + set flist [list] + proj-assert {$dirExt eq ""} + lappend flist $dirBld/teaish.tcl $dirSrc/teaish.tcl if {![proj-first-file-found extT $flist]} { if {$gotHelpArg} { # Tell teaish-configure-core that the lack of extension is not - # an error when --help is used. + # an error when --help or --teaish-install is used. return 0; } proj-indented-notice -error " @@ -833,17 +928,14 @@ If you are attempting an out-of-tree build, use proj-error "extension tcl file is not readable: $extT" } set ::teaish__Config(teaish.tcl) $extT - - if {"" eq $dirExt} { - # If this wasn't set via --teaish-extension-dir then derive it from - # $extT. - #puts "extT=$extT dirExt=$dirExt" - set dirExt [file dirname $extT] - } + set dirExt [file dirname $extT] set ::teaish__Config(extension-dir) $dirExt set ::teaish__Config(blddir-is-extdir) [expr {$dirBld eq $dirExt}] set ::teaish__Config(dist-enabled) $::teaish__Config(blddir-is-extdir); # may change later + set ::teaish__Config(dist-full-enabled) \ + [expr {[file-normalize $::autosetup(srcdir)] + eq [file-normalize $::teaish__Config(extension-dir)]}] set addDist {{file} { teaish-dist-add [file tail $file] @@ -863,55 +955,65 @@ If you are attempting an out-of-tree build, use # $dirExt. # if {[proj-first-file-found extM \ - [list $dirExt/teaish.make.in $dirExt/teaish.make]]} { + [list \ + $dirExt/teaish.make.in \ + $dirExt/teaish.make \ + ]]} { if {[string match *.in $extM]} { define TEAISH_MAKEFILE_IN $extM define TEAISH_MAKEFILE [file rootname [file tail $extM]] - proj-dot-ins-append $extM [get-define TEAISH_MAKEFILE] } else { define TEAISH_MAKEFILE_IN "" define TEAISH_MAKEFILE $extM } apply $addDist $extM - teaish__verbose 1 msg-result "Extension makefile = $extM" + teaish__verbose 1 msg-result "Extension makefile = $extM" } else { define TEAISH_MAKEFILE_IN "" define TEAISH_MAKEFILE "" } # Look for teaish.pkginit.tcl[.in] + set piPolicy 0 if {[proj-first-file-found extI \ - [list $dirExt/teaish.pkginit.tcl.in $dirExt/teaish.pkginit.tcl]]} { + [list \ + $dirExt/teaish.pkginit.tcl.in \ + $dirExt/teaish.pkginit.tcl \ + ]]} { if {[string match *.in $extI]} { - proj-dot-ins-append $extI + # Generate teaish.pkginit.tcl from $extI. define TEAISH_PKGINIT_TCL_IN $extI - define TEAISH_PKGINIT_TCL [file tail [file rootname $extI]] + define TEAISH_PKGINIT_TCL [file rootname [file tail $extI]] + set piPolicy 0x01 } else { + # Assume static $extI. define TEAISH_PKGINIT_TCL_IN "" define TEAISH_PKGINIT_TCL $extI + set piPolicy 0x10 } apply $addDist $extI teaish__verbose 1 msg-result "Extension post-load init = $extI" define TEAISH_PKGINIT_TCL_TAIL \ [file tail [get-define TEAISH_PKGINIT_TCL]]; # for use in pkgIndex.tcl.in } + set ::teaish__Config(pkginit-policy) $piPolicy # Look for pkgIndex.tcl[.in]... set piPolicy 0 if {[proj-first-file-found extPI $dirExt/pkgIndex.tcl.in]} { - # Generate ./pkgIndex.tcl from it. + # Generate ./pkgIndex.tcl from $extPI. define TEAISH_PKGINDEX_TCL_IN $extPI define TEAISH_PKGINDEX_TCL [file rootname [file tail $extPI]] apply $addDist $extPI set piPolicy 0x01 } elseif {$dirExt ne $dirSrc && [proj-first-file-found extPI $dirSrc/pkgIndex.tcl.in]} { - # Generate ./pkgIndex.tcl from it. + # Generate ./pkgIndex.tcl from $extPI. define TEAISH_PKGINDEX_TCL_IN $extPI define TEAISH_PKGINDEX_TCL [file rootname [file tail $extPI]] set piPolicy 0x02 } elseif {[proj-first-file-found extPI $dirExt/pkgIndex.tcl]} { - # Assume it's a static file and use it. + # Assume $extPI's a static file and use it. define TEAISH_PKGINDEX_TCL_IN "" define TEAISH_PKGINDEX_TCL $extPI apply $addDist $extPI @@ -932,7 +1034,6 @@ If you are attempting an out-of-tree build, use file delete -force -- $xt; # ensure no stale copy is used define TEAISH_TEST_TCL $xt define TEAISH_TEST_TCL_IN $ttt - proj-dot-ins-append $ttt $xt } else { define TEAISH_TEST_TCL $ttt define TEAISH_TEST_TCL_IN "" @@ -951,18 +1052,21 @@ If you are attempting an out-of-tree build, use file delete -force -- $xt; # ensure no stale copy is used define TEAISH_TESTER_TCL $xt define TEAISH_TESTER_TCL_IN $ttt - proj-dot-ins-append $ttt $xt if {[lindex $flist 0] eq $ttt} { apply $addDist $ttt } + unset ttt xt } else { - set ttt [file join $dirSrc teaish.tester.tcl.in] - set xt [file rootname [file tail $ttt]] - proj-dot-ins-append $ttt $xt - define TEAISH_TESTER_TCL $xt - define TEAISH_TESTER_TCL_IN $ttt + if {[file exists [set ttt [file join $dirSrc teaish.tester.tcl.in]]]} { + set xt [file rootname [file tail $ttt]] + define TEAISH_TESTER_TCL $xt + define TEAISH_TESTER_TCL_IN $ttt + } else { + define TEAISH_TESTER_TCL "" + define TEAISH_TESTER_TCL_IN "" + } } - unset flist xt ttt + unset flist # TEAISH_OUT_OF_EXT_TREE = 1 if we're building from a dir other # than the extension's home dir. @@ -982,14 +1086,73 @@ proc teaish-cflags-add {args} { } # -# @teaish-define-to-cflag defineName... +# @teaish-define-to-cflag ?flags? defineName...|{defineName...} # # Uses [proj-define-to-cflag] to expand a list of [define] keys, each # one a separate argument, to CFLAGS-style -D... form then appends # that to the current TEAISH_CFLAGS. # +# It accepts these flags from proj-define-to-cflag: -quote, +# -zero-undef. It does _not_ support its -list flag. +# +# It accepts its non-flag argument(s) in 2 forms: (1) each arg is a +# single [define] key or (2) its one arg is a list of such keys. +# +# TODO: document teaish's well-defined (as it were) defines for this +# purpose. At a bare minimum: +# +# - TEAISH_NAME +# - TEAISH_PKGNAME +# - TEAISH_VERSION +# - TEAISH_LIBDIR_NAME +# - TEAISH_LOAD_PREFIX +# - TEAISH_URL +# proc teaish-define-to-cflag {args} { - teaish-cflags-add [proj-define-to-cflag {*}$args] + set flags {} + while {[string match -* [lindex $args 0]]} { + set arg [lindex $args 0] + switch -exact -- $arg { + -quote - + -zero-undef { + lappend flags $arg + set args [lassign $args -] + } + default break + } + } + if {1 == [llength $args]} { + set args [list {*}[lindex $args 0]] + } + #puts "***** flags=$flags args=$args" + teaish-cflags-add [proj-define-to-cflag {*}$flags {*}$args] +} + +# +# @teaish-cflags-for-tea ?...CFLAGS? +# +# Adds several -DPACKAGE_... CFLAGS using the extension's metadata, +# all as quoted strings. Those symbolic names are commonly used in +# TEA-based builds, and this function is intended to simplify porting +# of such builds. The -D... flags added are: +# +# -DPACKAGE_VERSION=... +# -DPACKAGE_NAME=... +# -DPACKAGE_URL=... +# -DPACKAGE_STRING=... +# +# Any arguments are passed-on as-is to teaish-cflags-add. +# +proc teaish-cflags-for-tea {args} { + set name $::teaish__PkgInfo(-name) + set version $::teaish__PkgInfo(-version) + set pstr [join [list $name $version]] + teaish-cflags-add \ + {*}$args \ + '-DPACKAGE_VERSION="$version"' \ + '-DPACKAGE_NAME="$name"' \ + '-DPACKAGE_STRING="$pstr"' \ + '-DPACKAGE_URL="[teaish-get -url]"' } # @@ -1020,17 +1183,18 @@ proc teaish-ldflags-prepend {args} { # # @teaish-src-add ?-dist? ?-dir? src-files... # -# Appends all non-empty $args to the project's list of C/C++ source -# files. +# Appends all non-empty $args to the project's list of C/C++ source or +# (in some cases) object files. # # If passed -dist then it also passes each filename, as-is, to # [teaish-dist-add]. # # If passed -dir then each src-file has [teaish-get -dir] prepended to -# it for before they're added to the list. As often as not, that will -# be the desired behavior so that out-of-tree builds can find the +# it before they're added to the list. As often as not, that will be +# the desired behavior so that out-of-tree builds can find the # sources, but there are cases where it's not desired (e.g. when using -# a source file from outside of the extension's dir). +# a source file from outside of the extension's dir, or when adding +# object files (which are typically in the build tree)). # proc teaish-src-add {args} { set i 0 @@ -1070,6 +1234,7 @@ proc teaish-dist-add {args} { # ^^^ reminder: we ignore $::teaish__Config(dist-enabled) here # because the client might want to implement their own dist # rules. + #proj-warn "**** args=$args" lappend ::teaish__Config(dist-files) {*}$args } } @@ -1089,18 +1254,19 @@ proc teaish-dist-add {args} { # -tab: emit a literal tab # -nl: emit a literal newline # -nltab: short for -nl -tab -# -eol: emit a backslash-escaped end-of-line -# -eoltab: short for -eol -tab +# -bnl: emit a backslash-escaped end-of-line +# -bnltab: short for -eol -tab # # Anything else is appended verbatim. This function adds no additional # spacing between each argument nor between subsequent invocations. -# +# Generally speaking, a series of calls to this function need to +# be sure to end the series with a newline. proc teaish-make-add {args} { set out [get-define TEAISH_MAKEFILE_CODE ""] foreach a $args { switch -exact -- $a { - -eol { set a " \\\n" } - -eoltab { set a " \\\n\t" } + -bnl { set a " \\\n" } + -bnltab { set a " \\\n\t" } -tab { set a "\t" } -nl { set a "\n" } -nltab { set a "\n\t" } @@ -1110,6 +1276,116 @@ proc teaish-make-add {args} { define TEAISH_MAKEFILE_CODE $out } +# Internal helper to generate a clean/distclean rule name +proc teaish__cleanup_rule {{tgt clean}} { + set x [incr ::teaish__Config(teaish__cleanup_rule-counter-${tgt})] + return ${tgt}-_${x}_ +} + +# @teaish-make-obj objfile srcfile ?...args? +# +# Uses teaish-make-add to inject makefile rules for $objfile from +# $srcfile, which is assumed to be C code which uses libtcl. Unless +# -recipe is used (see below) it invokes the compiler using the +# makefile-defined $(CC.tcl) which, in the default Makefile.in +# template, includes any flags needed for building against the +# configured Tcl. +# +# This always terminates the resulting code with a newline. +# +# Any arguments after the 2nd may be flags described below or, if no +# -recipe is provided, flags for the compiler call. +# +# -recipe {...} +# Uses the trimmed value of {...} as the recipe, prefixing it with +# a single hard-tab character. +# +# -deps {...} +# List of extra files to list as dependencies of $o. Good luck +# escaping non-trivial cases properly. +# +# -clean +# Generate cleanup rules as well. +proc teaish-make-obj {o src args} { + set consume 0 + set clean 0 + set flag "" + array set flags {} + set xargs {} + foreach arg $args { + if {$consume} { + set consume 0 + set flags($flag) $arg + continue + } + switch -exact -- $arg { + -clean {incr clean} + -recipe - + -deps { + set flag $arg + incr consume + } + default { + lappend xargs $arg + } + } + } + teaish-make-add \ + "# [proj-current-scope 1] -> [proj-current-scope] $o $src" -nl \ + "$o: $src $::teaish__Config(teaish.tcl)" + if {[info exists flags(-deps)]} { + teaish-make-add " " [join $flags(-deps)] + } + teaish-make-add -nltab + if {[info exists flags(-recipe)]} { + teaish-make-add [string trim $flags(-recipe)] -nl + } else { + teaish-make-add [join [list \$(CC.tcl) -c $src {*}$xargs]] -nl + } + if {$clean} { + set rule [teaish__cleanup_rule] + teaish-make-add \ + "clean: $rule\n$rule:\n\trm -f \"$o\"\n" + } +} + +# +# @teaish-make-clean ?-r? ?-dist? ...files|{...files} +# +# Adds makefile rules for cleaning up the given files via the "make +# clean" or (if -dist is used) "make distclean" makefile rules. The -r +# flag uses "rm -fr" instead of "rm -f", so be careful with that. +# +# The file names are taken literally as arguments to "rm", so they may +# be shell wildcards to be resolved at cleanup-time. To clean up whole +# directories, pass the -r flag. Each name gets quoted in +# double-quotes, so spaces in names should not be a problem (but +# double-quotes in names will be). +# +proc teaish-make-clean {args} { + if {1 == [llength $args]} { + set args [list {*}[lindex $args 0]] + } + + set tgt clean + set rmflags "-f" + proj-parse-simple-flags args flags { + -dist 0 { + set tgt distclean + } + -r 0 { + set rmflags "-fr" + } + } + set rule [teaish__cleanup_rule $tgt] + teaish-make-add "# [proj-current-scope 1] -> [proj-current-scope]: [join $args]\n" + teaish-make-add "${rule}:\n\trm ${rmflags}" + foreach a $args { + teaish-make-add " \"$a\"" + } + teaish-make-add "\n${tgt}: ${rule}\n" +} + # # @teaish-make-config-header filename # @@ -1344,7 +1620,7 @@ proc teaish__dump_defs_to_list {args} { } # -# @teaish__pragma ...flags +# teaish__pragma ...flags # # Offers a way to tweak how teaish's core behaves in some cases, in # particular those which require changing how the core looks for an @@ -1365,6 +1641,19 @@ proc teaish__dump_defs_to_list {args} { # no-dll [L]: tells teaish to elide the DLL-building recipe # from the generated Makefile. # +# no-vsatisfies-error [L]: tells teaish that failure to match the +# -vsatisfies value should simply "return" instead of "error". +# +# no-tester [L]: disables automatic generation of teaish.test.tcl +# even if a copy of teaish.tester.tcl.in is found. +# +# no-full-dist [L]: changes the "make dist" rules to never include +# a copy of teaish itself. By default it will include itself only +# if the extension lives in the same directory as teaish. +# +# full-dist [L]: changes the "make dist" rules to always include +# a copy of teaish itself. +# # Emits a warning message for unknown arguments. # proc teaish__pragma {args} { @@ -1386,10 +1675,27 @@ proc teaish__pragma {args} { set ::teaish__Config(dist-enabled) 0 } + full-dist { + set ::teaish__Config(dist-full-enabled) 1 + } + + no-full-dist { + set ::teaish__Config(dist-full-enabled) 0 + } + no-dll { set ::teaish__Config(dll-enabled) 0 } + no-vsatisfies-error { + set ::teaish__Config(vsatisfies-error) 0 + } + + no-tester { + define TEAISH_TESTER_TCL_IN "" + define TEAISH_TESTER_TCL "" + } + default { proj-error "Unknown flag: $arg" } @@ -1430,7 +1736,7 @@ proc teaish__pragma {args} { # directory containing the extension. (In TEA this would be the # PACKAGE_NAME, not to be confused with...) # -# -pkgName pkg-provide-name: The extension's name for purposes of +# -name.pkg pkg-provide-name: The extension's name for purposes of # Tcl_PkgProvide(), [package require], and friends. It defaults to # the `-name`, and is normally the same, but some projects (like # SQLite) have a different name here than they do in their @@ -1441,11 +1747,11 @@ proc teaish__pragma {args} { # # -libDir dirName: The base name of the directory into which this # extension should be installed. It defaults to a concatenation of -# `-pkgName` and `-version`. +# `-name.pkg` and `-version`. # # -loadPrefix prefix: For use as the second argument passed to # Tcl's `load` command in the package-loading process. It defaults -# to title-cased `-pkgName` because Tcl's `load` plugin system +# to title-cased `-name.pkg` because Tcl's `load` plugin system # expects it in that form. # # -options {...}: If provided, it must be a list compatible with @@ -1468,6 +1774,8 @@ proc teaish__pragma {args} { # order. Failure to meet a `vsatisfies` condition triggers an # error. # +# -url {...}: an optional URL for the extension. +# # -pragmas {...} A list of infrequently-needed lower-level # directives which can influence teaish, including: # @@ -1485,10 +1793,26 @@ proc teaish__pragma {args} { # support script-only extensions. # # Unsupported flags or pragmas will trigger an error. +# +# Potential pothole: setting certain state, e.g. -version, after the +# initial call requires recalculating of some [define]s. Any such +# changes should be made as early as possible in teaish-configure so +# that any later use of those [define]s gets recorded properly (not +# with the old value). This is particularly relevant when it is not +# possible to determine the -version or -name until teaish-configure +# has been called, and it's updated dynamically from +# teaish-configure. Notably: +# +# - If -version or -name are updated, -libDir will almost certainly +# need to be explicitly set along with them. +# +# - If -name is updated, -loadPrefix probably needs to be as well. +# proc teaish-pkginfo-set {args} { set doVars 0 set doCommands 0 set xargs $args + set recalc {} foreach arg $args { switch -exact -- $arg { -vars { @@ -1529,30 +1853,58 @@ proc teaish-pkginfo-set {args} { proj-error -up "Too many (or unknown) arguments to [proj-current-scope]: $args" } foreach {f d} $::teaish__Config(pkginfo-f2d) { - if {$sentinel ne [set v $flags($f)]} { - switch -exact -- $f { - -options { - proj-assert {"" eq $d} - options-add $v - } - -pragmas { - foreach p $v { - teaish__pragma $p - } - } - -vsatisfies { - if {1 == [llength $v] && 1 == [llength [lindex $v 0]]} { - # Transform X to {Tcl $X} - set v [list [join [list Tcl $v]]] - } - define $d $v - } - default { - define $d $v + if {$sentinel eq [set v $flags($f)]} continue + switch -exact -- $f { + -options { + proj-assert {"" eq $d} + options-add $v + } + -pragmas { + foreach p $v { + teaish__pragma $p } } - set ::teaish__PkgInfo($f) $v + -vsatisfies { + if {1 == [llength $v] && 1 == [llength [lindex $v 0]]} { + # Transform X to {Tcl $X} + set v [list [join [list Tcl $v]]] + } + define $d $v + } + -pkgInit.tcl.in { + # Generate pkginit file X from X.in + set ::teaish__Config(pkginit-policy) 0x02 + set x [file join $::teaish__Config(extension-dir) $v] + define TEAISH_PKGINIT_TCL_IN $x + set fout [file rootname [file tail $v]] + define TEAISH_PKGINIT_TCL $fout + define TEAISH_PKGINIT_TCL_TAIL $fout + set ::teaish__PkgInfo(-pkgInit.tcl) {} + teaish-dist-add $v + set v $x + } + -pkgInit.tcl { + # Static pkginit file X + set ::teaish__Config(pkginit-policy) 0x20 + set x [file join $::teaish__Config(extension-dir) $v] + define TEAISH_PKGINIT_TCL $x + define TEAISH_PKGINIT_TCL_IN "" + define TEAISH_PKGINIT_TCL_TAIL [file tail $v] + set ::teaish__PkgInfo(-pkgInit.tcl.in) {} + teaish-dist-add $v + set v $x + } + default { + define $d $v + } } + set ::teaish__PkgInfo($f) $v + if {$f in {-name -version -libDir -loadPrefix}} { + lappend recalc $f + } + } + if {"" ne $recalc} { + teaish__define_pkginfo_derived $recalc } } @@ -1616,12 +1968,40 @@ proc teaish-pkginfo-get {args} { } } + set cases [join $cases] foreach {flag defName} $::teaish__Config(pkginfo-f2d) { - switch -exact -- $flag [join $cases] + switch -exact -- $flag $cases } if {0 == $argc} { return $rv } } +# (Re)set some defines based on pkginfo state. $flags is the list of +# pkginfo -flags which triggered this, or "*" for the initial call. +proc teaish__define_pkginfo_derived {flags} { + set all [expr {{*} in $flags}] + if {$all || "-version" in $flags || "-name" in $flags} { + set name $::teaish__PkgInfo(-name) ; # _not_ -name.pkg + if {[info exists ::teaish__PkgInfo(-version)]} { + set pkgver $::teaish__PkgInfo(-version) + set libname "lib" + if {[string match *-cygwin [get-define host]]} { + set libname cyg + } + define TEAISH_DLL8_BASENAME $libname$name$pkgver + define TEAISH_DLL9_BASENAME ${libname}tcl9$name$pkgver + set ext [get-define TARGET_DLLEXT] + define TEAISH_DLL8 [get-define TEAISH_DLL8_BASENAME]$ext + define TEAISH_DLL9 [get-define TEAISH_DLL9_BASENAME]$ext + } + } + if {$all || "-libDir" in $flags} { + if {[info exists ::teaish__PkgInfo(-libDir)]} { + define TCLLIBDIR \ + [file dirname [get-define TCLLIBDIR]]/$::teaish__PkgInfo(-libDir) + } + } +} + # # @teaish-checks-queue -pre|-post args... # @@ -1680,6 +2060,7 @@ proc teaish-checks-run {flag} { # Triggers an error if passed an unknown flag. # proc teaish-get {flag} { + #-teaish.tcl {return $::teaish__Config(teaish.tcl)} switch -exact -- $flag { -dir { return $::teaish__Config(extension-dir) @@ -1690,9 +2071,6 @@ proc teaish-get {flag} { -build-dir { return $::autosetup(builddir) } - -teaish.tcl { - return $::teaish__Config(teaish.tcl) - } default { if {[info exists ::teaish__PkgInfo($flag)]} { return $::teaish__PkgInfo($flag) @@ -1702,6 +2080,12 @@ proc teaish-get {flag} { proj-error "Unhandled flag: $flag" } +array set ::teaish__PkgInfoF2D $::teaish__Config(pkginfo-f2d) +proc teaish__f2d_array {tgtArrayName} { + upvar $tgtArrayName tgt + set tgt $::teaish__PkgInfoF2D +} + # # Handles --teaish-create-extension=TARGET-DIR # @@ -1710,15 +2094,15 @@ proc teaish__create_extension {dir} { if {"" eq $dir} { proj-error "--teaish-create-extension=X requires a directory name." } - file mkdir $dir + file mkdir $dir/generic set cwd [pwd] #set dir [file-normalize [file join $cwd $dir]] - msg-result "Created dir $dir" + teaish__verbose 1 msg-result "Created dir $dir" cd $dir if {!$force} { # Ensure that we don't blindly overwrite anything foreach f { - teaish.c + generic/teaish.c teaish.tcl teaish.make.in teaish.test.tcl @@ -1733,15 +2117,35 @@ proc teaish__create_extension {dir} { set pkgName $name set version 0.0.1 set loadPrefix [string totitle $pkgName] - set content "teaish-pkginfo-set \ - -name ${name} \ - -pkgName ${pkgName} \ - -version ${version} \ - -loadPrefix $loadPrefix \ - -libDir ${name}${version} - -vsatisfies {{Tcl 8.5-}} \ - -options { foo=1 => {Disable foo} } - + set content {teaish-pkginfo-set } + #puts "0 content=$content" + if {[opt-str teaish-extension-pkginfo epi]} { + set epi [string trim $epi] + if {[string match "*\n*" $epi]} { + set epi "{$epi}" + } elseif {![string match "{*}" $epi]} { + append content "\{" $epi "\}" + } else { + append content $epi + } + #puts "2 content=$content\nepi=$epi" + } else { + append content [subst -nocommands -nobackslashes {{ + -name ${name} + -name.pkg ${pkgName} + -name.dist ${pkgName} + -version ${version} + -loadPrefix $loadPrefix + -libDir ${name}${version} + -vsatisfies {{Tcl 8.5-}} + -url {} + -options {} + -pragmas {full-dist} + }}] + #puts "3 content=$content" + } + #puts "1 content=$content" + append content "\n" { #proc teaish-options {} { # Return a list and/or use \[options-add\] to add new # configure flags. This is called before teaish's @@ -1757,13 +2161,18 @@ proc teaish__create_extension {dir} { # not be defined. #} +# Called by teaish once bootstrapping is complete. +# This function is responsible for the client-specific +# parts of the configuration process. proc teaish-configure {} { - teaish-src-add -dir -dist teaish.c + teaish-src-add -dir -dist generic/teaish.c + teaish-define-to-cflag -quote TEAISH_PKGNAME TEAISH_VERSION + # TODO: your code goes here.. } -" +}; # $content proj-file-write teaish.tcl $content - msg-result "Created teaish.tcl" + teaish__verbose 1 msg-result "Created teaish.tcl" set content "# Teaish test script. # When this tcl script is invoked via 'make test' it will have loaded @@ -1771,7 +2180,7 @@ proc teaish-configure {} { # autosetup/teaish/tester.tcl. " proj-file-write teaish.test.tcl $content - msg-result "Created teaish.test.tcl" + teaish__verbose 1 msg-result "Created teaish.test.tcl" set content [subst -nocommands -nobackslashes { #include @@ -1785,25 +2194,130 @@ extern int DLLEXPORT ${loadPrefix}_Init(Tcl_Interp *interp){ if (Tcl_InitStubs(interp, TCL_VERSION, 0) == NULL) { return TCL_ERROR; } - if (Tcl_PkgProvide(interp, "${name}", "${version}") == TCL_ERROR) { + if (Tcl_PkgProvide(interp, TEAISH_PKGNAME, TEAISH_VERSION) == TCL_ERROR) { return TCL_ERROR; } - Tcl_CreateObjCommand(interp, "${name}", ${loadPrefix}_Cmd, NULL, NULL); + Tcl_CreateObjCommand(interp, TEAISH_PKGNAME, ${loadPrefix}_Cmd, NULL, NULL); return TCL_OK; } }] - proj-file-write teaish.c $content - msg-result "Created teaish.c" + proj-file-write generic/teaish.c $content + teaish__verbose 1 msg-result "Created generic/teaish.c" set content "# teaish makefile for the ${name} extension -# tx.src = \$(tx.dir)/teaish.c +# tx.src = \$(tx.dir)/generic/teaish.c # tx.LDFLAGS = # tx.CFLAGS = " proj-file-write teaish.make.in $content - msg-result "Created teaish.make.in" + teaish__verbose 1 msg-result "Created teaish.make.in" - msg-result "Created new extension $name in \[$dir]" + msg-result "Created new extension $name in \[$dir]." cd $cwd + set ::teaish__Config(install-ext-dir) $dir +} + +# +# Internal helper for teaish__install +# +proc teaish__install_file {f destDir force} { + set dest $destDir/[file tail $f] + if {[file isdirectory $f]} { + file mkdir $dest + } elseif {!$force && [file exists $dest]} { + array set st1 [file stat $f] + array set st2 [file stat $dest] + if {($st1(mtime) == $st2(mtime)) + && ($st1(size) == $st2(size))} { + if {[file tail $f] in { + pkgIndex.tcl.in + teaish.tester.tcl.in + }} { + # Assume they're the same. In the scope of the "make dist" + # rules, this happens legitimately when an extension with a + # copy of teaish installed in the same dir assumes that the + # pkgIndex.tcl.in and teaish.tester.tcl.in belong to the + # extension, whereas teaish believes they belong to teaish. + # So we end up with dupes of those. + return + } + } + proj-error -up "Cowardly refusing to overwrite \[$dest\]." \ + "Use --teaish-force to enable overwriting." + } else { + # file copy -force $f $destDir; # loses +x bit + # + # JimTcl doesn't have [file attribute], so we can't use that here + # (in the context of an autosetup configure script). + exec cp -p $f $dest + } +} + +# +# Installs a copy of teaish, with autosetup, to $dDest, which defaults +# to the --teaish-install=X or --teash-create-extension=X dir. Won't +# overwrite files unless --teaish-force is used. +# +proc teaish__install {{dDest ""}} { + if {$dDest in {auto ""}} { + set dDest [opt-val teaish-install] + if {$dDest in {auto ""}} { + if {[info exists ::teaish__Config(install-ext-dir)]} { + set dDest $::teaish__Config(install-ext-dir) + } + } + } + set force [opt-bool teaish-force] + if {$dDest in {auto ""}} { + proj-error "Cannot determine installation directory." + } elseif {!$force && [file exists $dDest/auto.def]} { + proj-error \ + "Target dir looks like it already contains teaish and/or autosetup: $dDest" \ + "\nUse --teaish-force to overwrite it." + } + + set dSrc $::autosetup(srcdir) + set dAS $::autosetup(libdir) + set dAST $::teaish__Config(core-dir) + set dASTF $dAST/feature + teaish__verbose 1 msg-result "Installing teaish to \[$dDest\]..." + if {$::teaish__Config(verbose)>1} { + msg-result "dSrc = $dSrc" + msg-result "dAS = $dAS" + msg-result "dAST = $dAST" + msg-result "dASTF = $dASTF" + msg-result "dDest = $dDest" + } + + # Dest subdirs... + set ddAS $dDest/autosetup + set ddAST $ddAS/teaish + set ddASTF $ddAST/feature + foreach {srcDir destDir} [list \ + $dAS $ddAS \ + $dAST $ddAST \ + $dASTF $ddASTF \ + ] { + teaish__verbose 1 msg-result "Copying files to $destDir..." + file mkdir $destDir + foreach f [glob -directory $srcDir *] { + if {[string match {*~} $f] || [string match "#*#" [file tail $f]]} { + # Editor-generated backups and emacs lock files + continue + } + teaish__verbose 2 msg-result "\t$f" + teaish__install_file $f $destDir $force + } + } + teaish__verbose 1 msg-result "Copying files to $dDest..." + foreach f { + auto.def configure Makefile.in pkgIndex.tcl.in + teaish.tester.tcl.in + } { + teaish__verbose 2 msg-result "\t$f" + teaish__install_file $dSrc/$f $dDest $force + } + set ::teaish__Config(install-self-dir) $dDest + msg-result "Teaish $::teaish__Config(version) installed in \[$dDest\]." } diff --git a/autoconf/tea/autosetup/tester.tcl b/autoconf/tea/autosetup/tester.tcl index 5f5e1f1555..5c546e841d 100644 --- a/autoconf/tea/autosetup/tester.tcl +++ b/autoconf/tea/autosetup/tester.tcl @@ -115,12 +115,15 @@ proc test-assert {testId script {msg ""}} { # @test-expect testId script result # # Runs $script in the calling scope and compares its result to -# $result. If they differ, it triggers an [assert]. +# $result, minus any leading or trailing whitespace. If they differ, +# it triggers an [assert]. # proc test-expect {testId script result} { puts "test $testId" - set x [uplevel 1 $script] - test__assert 1 {$x eq $result} "\nEXPECTED: <<$result>>\nGOT: <<$x>>" + set x [string trim [uplevel 1 $script]] + set result [string trim $result] + test__assert 1 {$x eq $result} \ + "\nEXPECTED: <<$result>>\nGOT: <<$x>>" } # diff --git a/autoconf/tea/teaish.tcl b/autoconf/tea/teaish.tcl index 99db68e249..87d059c328 100644 --- a/autoconf/tea/teaish.tcl +++ b/autoconf/tea/teaish.tcl @@ -1,30 +1,74 @@ # Teaish configure script for the SQLite Tcl extension -apply {{} { - set version [proj-file-content -trim [teaish-get -dir]/../VERSION] - proj-assert {[string match 3.*.* $version]} - teaish-pkginfo-set -vars { - -name sqlite - -pkgName sqlite3 - -version $version - -loadPrefix Sqlite3 - -vsatisfies 8.6- - -libDir sqlite$version - -pragmas {no-dist} - } -}} - # -# Object for communicating certain config-time state across various -# auto.def-related pieces. +# State for disparate config-time pieces. # -array set sqliteConfig [subst [proj-strip-hash-comments { +array set sqlite__Config [proj-strip-hash-comments { # # The list of feature --flags which the --all flag implies. This # requires special handling in a few places. # all-flag-enables {fts3 fts4 fts5 rtree geopoly} -}]] + + # >0 if building in the canonical tree. -1=undetermined + is-canonical -1 +}] + +# +# Set up the package info for teaish... +# +apply {{dir} { + # Figure out the version number... + set version "" + if {[file exists $dir/../VERSION]} { + # The canonical SQLite TEA(ish) build + set version [proj-file-content -trim $dir/../VERSION] + set ::sqlite__Config(is-canonical) 1 + set distname sqlite-tcl + } elseif {[file exists $dir/generic/tclsqlite3.c]} { + # The copy from the teaish tree, used as a dev/test bed before + # updating SQLite's tree. + set ::sqlite__Config(is-canonical) 0 + set fd [open $dir/generic/tclsqlite3.c rb] + while {[gets $fd line] >=0} { + if {[regexp {^#define[ ]+SQLITE_VERSION[ ]+"(3.+)"} \ + $line - version]} { + set distname sqlite-teaish + break + } + } + close $fd + } + + if {"" eq $version} { + proj-fatal "Cannot determine the SQLite version number" + } + + proj-assert {$::sqlite__Config(is-canonical) > -1} + proj-assert {[string match 3.*.* $version]} \ + "Unexpected SQLite version: $version" + + set pragmas {} + if {$::sqlite__Config(is-canonical)} { + # Disable "make dist" in the canonical tree. That tree is + # generated from several pieces and creating/testing working + # "dist" rules for that sub-build currently feels unnecessary. The + # copy in the teaish tree, though, should be able to "make dist". + lappend pragmas no-dist + } else { + lappend pragmas full-dist + } + + teaish-pkginfo-set -vars { + -name sqlite + -name.pkg sqlite3 + -version $version + -name.dist $distname + -vsatisfies 8.6- + -libDir sqlite$version + -pragmas $pragmas + } +}} [teaish-get -dir] # # Must return either an empty string or a list in the form accepted by @@ -52,7 +96,7 @@ proc teaish-options {} { geopoly => {Enable the GEOPOLY extension} rtree => {Enable the RTREE extension} session => {Enable the SESSION extension} - all=1 => {Disable $::sqliteConfig(all-flag-enables)} + all=1 => {Disable $::sqlite__Config(all-flag-enables)} with-icu-ldflags:LDFLAGS => {Enable SQLITE_ENABLE_ICU and add the given linker flags for the ICU libraries. e.g. on Ubuntu systems, try '-licui18n -licuuc -licudata'.} @@ -75,8 +119,8 @@ proc teaish-options {} { proc teaish-configure {} { use teaish/feature-tests - set srcdir [teaish-get -dir] teaish-src-add -dist -dir generic/tclsqlite3.c + if {[proj-opt-was-provided override-sqlite-version]} { teaish-pkginfo-set -version [opt-val override-sqlite-version] proj-warn "overriding sqlite version number:" [teaish-pkginfo-get -version] @@ -105,8 +149,8 @@ proc teaish-configure {} { msg-result "Using system-level sqlite3." teaish-cflags-add -DUSE_SYSTEM_SQLITE teaish-ldflags-add -lsqlite3 - } else { - teaish-cflags-add -I${srcdir}/.. + } elseif {$::sqlite__Config(is-canonical)} { + teaish-cflags-add -I[teaish-get -dir]/.. } teaish-check-librt @@ -157,7 +201,7 @@ proc sqlite-handle-common-feature-flags {} { msg-result "Feature flags..." if {![opt-bool all]} { # Special handling for --disable-all - foreach flag $::sqliteConfig(all-flag-enables) { + foreach flag $::sqlite__Config(all-flag-enables) { if {![proj-opt-was-provided $flag]} { proj-opt-set $flag 0 } @@ -178,7 +222,7 @@ proc sqlite-handle-common-feature-flags {} { # The --geopoly flag, though, will automatically re-enable # --rtree, so --disable-rtree won't actually disable anything in # that case. - foreach k $::sqliteConfig(all-flag-enables) { + foreach k $::sqlite__Config(all-flag-enables) { if {![proj-opt-was-provided $k]} { proj-opt-set $k 1 } @@ -500,9 +544,6 @@ proc sqlite-munge-cflags {} { # -DSQLITE_ENABLE* to OPT_FEATURE_FLAGS. This behavior is derived # from the pre-3.48 build. # - # Handling of CPPFLAGS, as well as removing ENABLE/OMIT from - # CFLAGS/CPPFLAGS, was missing in the 3.49.0 release as well. - # # If any configure flags for features are in conflict with # CFLAGS/CPPFLAGS-specified feature flags, all bets are off. There # are no guarantees about which one will take precedence. diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index bdc593cd43..7690486cf2 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -85,7 +85,7 @@ set proj__Config(isatty) [isatty? stdout] # proc proj-warn {args} { show-notices - puts stderr [join [list "WARNING:" {*}$args] " "] + puts stderr [join [list "WARNING: \[[proj-current-scope 1]\]: " {*}$args] " "] } # @@ -1762,8 +1762,8 @@ proc proj-validate-no-unresolved-ats {args} { # # @proj-first-file-found tgtVar fileList # -# Searches $fileList for an existing file. If one is found, its name is -# assigned to tgtVar and 1 is returned, else tgtVar is not modified +# Searches $fileList for an existing file. If one is found, its name +# is assigned to tgtVar and 1 is returned, else tgtVar is set to "" # and 0 is returned. # proc proj-first-file-found {tgtVar fileList} { @@ -1774,6 +1774,7 @@ proc proj-first-file-found {tgtVar fileList} { return 1 } } + set tgt "" return 0 } diff --git a/manifest b/manifest index 15e4fa0d6b..f6282811e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C proj.tcl:\scorrect\sthe\sauto-reconfigure\srules\sto\sinclude\sany\sX=Y\spassed\sto\sconfigure.\sImprove\shandling\sof\squoted\sCFLAGS. -D 2025-04-27T06:01:26.900 +C tea:\spull\sin\supstream\steaish.\sConsolidate\sthe\sbuild\sdefinition\sfrom\sthe\supstream\sbuild\sand\sthis\sone\sto\sease\smaintenace.\sImprove\scompatibility\swith\shistorical\sTEA\susage\sof\s--with-tcl=...\s(it\sinsists\son\sa\s/lib\ssuffix\sfor\sthat\spath). +D 2025-04-28T09:25:19.852 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -22,18 +22,18 @@ F autoconf/Makefile.msc 5bc67d3912444c40c6f96d003e5c90663e51abb83d204a520110b1b2 F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac -F autoconf/tea/Makefile.in bb66da485861af82a45e7c135fcf89270f5c0989dbfbb17912c745440b2f9c6c +F autoconf/tea/Makefile.in 6aebb3d820c8aab94f7599e8594d5f2ecea63894d08f36ba4141040b322042bc F autoconf/tea/README.txt 656d4686c509d375f5988ff3deda94f65fe6cd8358cd55d1f1dcc7b6e2ff73aa F autoconf/tea/auto.def ce95b9450e2fa4ba5dc857e208fe10f4e6f2d737796ac3278aee6079db417529 F autoconf/tea/autosetup/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca -F autoconf/tea/autosetup/core.tcl 603506c76d1bdf3cf12be0ed87e11fb77aeae3b0598b97e3989e6fabb60de01a +F autoconf/tea/autosetup/core.tcl f0eebd4aa416df44f0c22f7be1a2b5b05e8fe4f64e53b0d06a21f328963c5109 F autoconf/tea/autosetup/feature-tests.tcl 307cac35806817fc87bd0d92004364ee682c0e99f2ab505291500493ab6c7a5f -F autoconf/tea/autosetup/tester.tcl 93cff35afc5cabc8f855fad114fae58b435fa47cbab5f8cbd4dc694ff0c26107 +F autoconf/tea/autosetup/tester.tcl c293695a0ab5d9e8d0ceeb0ee422f90e8a6aa9f0c7c51acd0b6d9f09d8edfed3 F autoconf/tea/configure d0b12b984edca6030d1976375b80157ac78b5b90a5b4f0dcee39357f63f4a80b x F autoconf/tea/doc/sqlite3.n 9a97f4f717ceab73004ea412af7960625c1cb24b5c25e4ae4c8b5d8fa4300f4e F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in e07da6b94561f4aa382bab65b1ccceb04701b97bf59d007c1d1f20a222b22d07 -F autoconf/tea/teaish.tcl 172fd9d6e06e5874db58e3384cfa9a7d52325634302eed189bb60f6cbf40bbd9 +F autoconf/tea/teaish.tcl 8e124f33cbaf9309f3e49be4e7018a03b8f3a52f8c8d9e1e5419f4f7b0eae59e F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 F autoconf/tea/teaish.tester.tcl.in 0d048e5569ad9bbaffbe5123c2084f9084d424a7719d06f5ca941caafbcca320 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 @@ -51,7 +51,7 @@ F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e F autosetup/jimsh0.c 563b966c137a4ce3c9333e5196723b7ac0919140a9d7989eb440463cd855c367 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl d2c0c6da22b4b6e0da6d054851079477815f8acc265e718f16879f3e88ffd819 +F autosetup/proj.tcl 4deaa5bba71390367a55fe4f0dbdc0e0ddbefb5c24a0ee4a5a54bbf288a3285d F autosetup/sqlite-config.tcl 54c88abadda1a42c6276186afcf6534e9bcb08350d96676c4ec08b99f2ada484 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x @@ -2216,8 +2216,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4aeec30443b282f10353cdb9415fdce436287280f0f5d5d6b9917da831744898 -R d75d037808a96903ea777bfc48335881 +P 7a2f4f319c90b768c6acc96671a3173c0ef0493180f3c4d5443f1ca03d92f6b4 +R 9db6b397f54d671a4320cc5ce752e15a U stephan -Z 7e9da962d1e2273ff3dad52e3e71aeeb +Z a79c230793c16373c3792db45c1c451d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6428295eb6..9d9b32d403 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a2f4f319c90b768c6acc96671a3173c0ef0493180f3c4d5443f1ca03d92f6b4 +0ea5993dd8549056d73f3d5447e0a810495ece23a75be8add78fba6842125d85 From 9112717180ec8bfa0fe1b61a83f51b5e2a3f614c Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Apr 2025 13:32:46 +0000 Subject: [PATCH 13/42] Remove long-obsolete performance testing scripts from the tools/ folder. The test/speedtest.tcl script is what should be used these days. This is just cruft clean-up. FossilOrigin-Name: 956af8c294ae5b826a5b645922df07dad360290d19bf2b6a7f74d0a783a3e692 --- manifest | 21 +--- manifest.uuid | 2 +- tool/kvtest-speed.sh | 35 ------ tool/mkspeedsql.tcl | 237 ----------------------------------- tool/run-speed-test.sh | 90 -------------- tool/speed-check.sh | 219 -------------------------------- tool/speedtest.tcl | 275 ----------------------------------------- tool/speedtest16.c | 171 ------------------------- tool/speedtest2.tcl | 207 ------------------------------- tool/speedtest8.c | 260 -------------------------------------- tool/speedtest8inst1.c | 218 -------------------------------- 11 files changed, 7 insertions(+), 1728 deletions(-) delete mode 100644 tool/kvtest-speed.sh delete mode 100644 tool/mkspeedsql.tcl delete mode 100644 tool/run-speed-test.sh delete mode 100644 tool/speed-check.sh delete mode 100644 tool/speedtest.tcl delete mode 100644 tool/speedtest16.c delete mode 100644 tool/speedtest2.tcl delete mode 100644 tool/speedtest8.c delete mode 100644 tool/speedtest8inst1.c diff --git a/manifest b/manifest index f6282811e9..990637bbe5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C tea:\spull\sin\supstream\steaish.\sConsolidate\sthe\sbuild\sdefinition\sfrom\sthe\supstream\sbuild\sand\sthis\sone\sto\sease\smaintenace.\sImprove\scompatibility\swith\shistorical\sTEA\susage\sof\s--with-tcl=...\s(it\sinsists\son\sa\s/lib\ssuffix\sfor\sthat\spath). -D 2025-04-28T09:25:19.852 +C Remove\slong-obsolete\sperformance\stesting\sscripts\sfrom\sthe\stools/\sfolder.\nThe\stest/speedtest.tcl\sscript\sis\swhat\sshould\sbe\sused\sthese\sdays.\s\sThis\nis\sjust\scruft\sclean-up. +D 2025-04-28T13:32:46.678 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2143,7 +2143,6 @@ F tool/genfkey.README e550911fa984c8255ebed2ef97824125d83806eb5232582700de949edf F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5 -F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c f51a488369046cd4f4212d755a214a57673ded400cbeb01e298cbf63539e1d8c F tool/lempar.c bdffd3b233a4e4e78056c9c01fadd2bb3fe902435abde3bce3d769fdf0d5cca2 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 @@ -2163,7 +2162,6 @@ F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07 F tool/mkpragmatab.tcl 3801ce32f8c55fe63a3b279f231fb26c2c1a2ea9a09d2dd599239d87a609acec F tool/mkshellc.tcl 9ce74de0fa904a2c56a96f8d8b5261246bacb0eaa8d7e184f9e18ff94145ebbc F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 -F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 351c55256213154cabb051a3c870ef9f4487de905015141ae50dc7578a901b84 F tool/mksqlite3c.tcl f11b63445c4840509248bd4aa151a81aea25d5415fef71943c8d436eba4f3b3c F tool/mksqlite3h.tcl 989948c6a26e188e673d7c2f2f093ea3acd816ad6ac65bab596280075c8f3a45 @@ -2179,7 +2177,6 @@ F tool/pagesig.c f98909b4168d9cac11a2de7f031adea0e2f3131faa7515a72807c03ec58eafe F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82d9a F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/run-speed-test.sh df9686c0991ea7c617b2cb5467d89d34b561f198ab91cb87735e27030ede92e8 F tool/showdb.c 3956d71e5193162609a60e8c9edfcf09274c00cfea2b1d221261427adb2b5cca F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9cc5e66d4ebbf2d194f39db2527ece92077e86ae627ddd233ee48e16e8142564 @@ -2188,12 +2185,6 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c F tool/showwal.c 11eca547980a066b081f512636151233350ac679f29ecf4ebfce7f4530230b3d F tool/soak1.tcl a3892082ed1079671565c044e93b55c3c7f38829aedf53cc597c65d23ffdaddf F tool/spaceanal.tcl 1f83962090a6b60e1d7bf92495d643e622bef9fe82ea3f2d22350dcbce9a12d0 -F tool/speed-check.sh 2d9e337449f8eb9f5ab4c1ce7433024e334ea03a68d48aa9caee6229c7cf0774 -F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 -F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e -F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff -F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 -F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 @@ -2216,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7a2f4f319c90b768c6acc96671a3173c0ef0493180f3c4d5443f1ca03d92f6b4 -R 9db6b397f54d671a4320cc5ce752e15a -U stephan -Z a79c230793c16373c3792db45c1c451d +P 0ea5993dd8549056d73f3d5447e0a810495ece23a75be8add78fba6842125d85 +R e9a7a5bb7d30c86d12fb872b44065ee2 +U drh +Z faa21c2c7bc49d38601dda410afe3656 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9d9b32d403..a89eb6ee7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ea5993dd8549056d73f3d5447e0a810495ece23a75be8add78fba6842125d85 +956af8c294ae5b826a5b645922df07dad360290d19bf2b6a7f74d0a783a3e692 diff --git a/tool/kvtest-speed.sh b/tool/kvtest-speed.sh deleted file mode 100644 index 5f2c8345be..0000000000 --- a/tool/kvtest-speed.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -# A script for running speed tests using kvtest. -# -# The test database must be set up first. Recommended -# command-line: -# -# ./kvtest init kvtest.db --count 100K --size 12K --variance 5K - -if test "$1" = "" -then - echo "Usage: $0 OUTPUTFILE [OPTIONS]" - exit -fi -NAME=$1 -shift -OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DIRECT_OVERFLOW_READ -DUSE_PREAD" -KVARGS="--count 100K --stats" -gcc -g -Os -I. $OPTS $* kvtest.c sqlite3.c -o kvtest - -# First run using SQL -rm cachegrind.out.[1-9][0-9]* -valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS 2>&1 | tee summary-kvtest-$NAME.txt -mv cachegrind.out.[1-9][0-9]* cachegrind.out.sql-$NAME -cg_anno.tcl cachegrind.out.sql-$NAME >cout-kvtest-sql-$NAME.txt - -# Second run using the sqlite3_blob object -valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS --blob-api 2>&1 | tee -a summary-kvtest-$NAME.txt -mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME -cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-$NAME.txt - -# Diff the sqlite3_blob API analysis for non-trunk runs. -if test "$NAME" != "trunk"; then - fossil test-diff --tk cout-kvtest-trunk.txt cout-kvtest-$NAME.txt & -fi diff --git a/tool/mkspeedsql.tcl b/tool/mkspeedsql.tcl deleted file mode 100644 index 04bafc04c1..0000000000 --- a/tool/mkspeedsql.tcl +++ /dev/null @@ -1,237 +0,0 @@ -# 2008 October 9 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# This file generates SQL text used for performance testing. -# -# $Id: mkspeedsql.tcl,v 1.1 2008/10/09 17:57:34 drh Exp $ -# - -# Set a uniform random seed -expr srand(0) - -# The number_name procedure below converts its argment (an integer) -# into a string which is the English-language name for that number. -# -# Example: -# -# puts [number_name 123] -> "one hundred twenty three" -# -set ones {zero one two three four five six seven eight nine - ten eleven twelve thirteen fourteen fifteen sixteen seventeen - eighteen nineteen} -set tens {{} ten twenty thirty forty fifty sixty seventy eighty ninety} -proc number_name {n} { - if {$n>=1000} { - set txt "[number_name [expr {$n/1000}]] thousand" - set n [expr {$n%1000}] - } else { - set txt {} - } - if {$n>=100} { - append txt " [lindex $::ones [expr {$n/100}]] hundred" - set n [expr {$n%100}] - } - if {$n>=20} { - append txt " [lindex $::tens [expr {$n/10}]]" - set n [expr {$n%10}] - } - if {$n>0} { - append txt " [lindex $::ones $n]" - } - set txt [string trim $txt] - if {$txt==""} {set txt zero} - return $txt -} - -# Create a database schema. -# -puts { - PRAGMA page_size=1024; - PRAGMA cache_size=8192; - PRAGMA locking_mode=EXCLUSIVE; - CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT); - CREATE TABLE t2(a INTEGER, b INTEGER, c TEXT); - CREATE INDEX i2a ON t2(a); - CREATE INDEX i2b ON t2(b); - SELECT name FROM sqlite_master ORDER BY 1; -} - - -# 50000 INSERTs on an unindexed table -# -set t1c_list {} -puts {BEGIN;} -for {set i 1} {$i<=50000} {incr i} { - set r [expr {int(rand()*500000)}] - set x [number_name $r] - lappend t1c_list $x - puts "INSERT INTO t1 VALUES($i,$r,'$x');" -} -puts {COMMIT;} - -# 50000 INSERTs on an indexed table -# -puts {BEGIN;} -for {set i 1} {$i<=50000} {incr i} { - set r [expr {int(rand()*500000)}] - puts "INSERT INTO t2 VALUES($i,$r,'[number_name $r]');" -} -puts {COMMIT;} - - -# 50 SELECTs on an integer comparison. There is no index so -# a full table scan is required. -# -for {set i 0} {$i<50} {incr i} { - set lwr [expr {$i*100}] - set upr [expr {($i+10)*100}] - puts "SELECT count(*), avg(b) FROM t1 WHERE b>=$lwr AND b<$upr;" -} - -# 50 SELECTs on an LIKE comparison. There is no index so a full -# table scan is required. -# -for {set i 0} {$i<50} {incr i} { - puts "SELECT count(*), avg(b) FROM t1 WHERE c LIKE '%[number_name $i]%';" -} - -# Create indices -# -puts {BEGIN;} -puts { - CREATE INDEX i1a ON t1(a); - CREATE INDEX i1b ON t1(b); - CREATE INDEX i1c ON t1(c); -} -puts {COMMIT;} - -# 5000 SELECTs on an integer comparison where the integer is -# indexed. -# -set sql {} -for {set i 0} {$i<5000} {incr i} { - set lwr [expr {$i*100}] - set upr [expr {($i+10)*100}] - puts "SELECT count(*), avg(b) FROM t1 WHERE b>=$lwr AND b<$upr;" -} - -# 100000 random SELECTs against rowid. -# -for {set i 1} {$i<=100000} {incr i} { - set id [expr {int(rand()*50000)+1}] - puts "SELECT c FROM t1 WHERE rowid=$id;" -} - -# 100000 random SELECTs against a unique indexed column. -# -for {set i 1} {$i<=100000} {incr i} { - set id [expr {int(rand()*50000)+1}] - puts "SELECT c FROM t1 WHERE a=$id;" -} - -# 50000 random SELECTs against an indexed column text column -# -set nt1c [llength $t1c_list] -for {set i 0} {$i<50000} {incr i} { - set r [expr {int(rand()*$nt1c)}] - set c [lindex $t1c_list $i] - puts "SELECT c FROM t1 WHERE c='$c';" -} - - -# Vacuum -puts {VACUUM;} - -# 5000 updates of ranges where the field being compared is indexed. -# -puts {BEGIN;} -for {set i 0} {$i<5000} {incr i} { - set lwr [expr {$i*2}] - set upr [expr {($i+1)*2}] - puts "UPDATE t1 SET b=b*2 WHERE a>=$lwr AND a<$upr;" -} -puts {COMMIT;} - -# 50000 single-row updates. An index is used to find the row quickly. -# -puts {BEGIN;} -for {set i 0} {$i<50000} {incr i} { - set r [expr {int(rand()*500000)}] - puts "UPDATE t1 SET b=$r WHERE a=$i;" -} -puts {COMMIT;} - -# 1 big text update that touches every row in the table. -# -puts { - UPDATE t1 SET c=a; -} - -# Many individual text updates. Each row in the table is -# touched through an index. -# -puts {BEGIN;} -for {set i 1} {$i<=50000} {incr i} { - set r [expr {int(rand()*500000)}] - puts "UPDATE t1 SET c='[number_name $r]' WHERE a=$i;" -} -puts {COMMIT;} - -# Delete all content in a table. -# -puts {DELETE FROM t1;} - -# Copy one table into another -# -puts {INSERT INTO t1 SELECT * FROM t2;} - -# Delete all content in a table, one row at a time. -# -puts {DELETE FROM t1 WHERE 1;} - -# Refill the table yet again -# -puts {INSERT INTO t1 SELECT * FROM t2;} - -# Drop the table and recreate it without its indices. -# -puts {BEGIN;} -puts { - DROP TABLE t1; - CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT); -} -puts {COMMIT;} - -# Refill the table yet again. This copy should be faster because -# there are no indices to deal with. -# -puts {INSERT INTO t1 SELECT * FROM t2;} - -# Select 20000 rows from the table at random. -# -puts { - SELECT rowid FROM t1 ORDER BY random() LIMIT 20000; -} - -# Delete 20000 random rows from the table. -# -puts { - DELETE FROM t1 WHERE rowid IN - (SELECT rowid FROM t1 ORDER BY random() LIMIT 20000); -} -puts {SELECT count(*) FROM t1;} - -# Delete 20000 more rows at random from the table. -# -puts { - DELETE FROM t1 WHERE rowid IN - (SELECT rowid FROM t1 ORDER BY random() LIMIT 20000); -} -puts {SELECT count(*) FROM t1;} diff --git a/tool/run-speed-test.sh b/tool/run-speed-test.sh deleted file mode 100644 index 9c76465a26..0000000000 --- a/tool/run-speed-test.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -# -# This is a template for a script used for day-to-day size and -# performance monitoring of SQLite. Typical usage: -# -# sh run-speed-test.sh trunk # Baseline measurement of trunk -# sh run-speed-test.sh x1 # Measure some experimental change -# fossil test-diff --tk cout-trunk.txt cout-x1.txt # View changes -# -# There are multiple output files, all with a base name given by -# the first argument: -# -# summary-$BASE.txt # Copy of standard output -# cout-$BASE.txt # cachegrind output -# explain-$BASE.txt # EXPLAIN listings (only with --explain) -# -if test "$1" = "" -then - echo "Usage: $0 OUTPUTFILE [OPTIONS]" - exit -fi -NAME=$1 -shift -CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5" -SPEEDTEST_OPTS="--shrink-memory --reprepare --heap 10000000 64" -SIZE=5 -doExplain=0 -while test "$1" != ""; do - case $1 in - --reprepare) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --autovacuum) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --utf16be) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --stats) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --without-rowid) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --nomemstat) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --wal) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal" - ;; - --size) - shift; SIZE=$1 - ;; - --explain) - doExplain=1 - ;; - --heap) - CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5" - shift; - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64" - ;; - *) - CC_OPTS="$CC_OPTS $1" - ;; - esac - shift -done -SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE" -echo "NAME = $NAME" | tee summary-$NAME.txt -echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt -echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt -rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o -gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c -size sqlite3.o | tee -a summary-$NAME.txt -if test $doExplain -eq 1; then - gcc -g -Os -Wall -I. $CC_OPTS \ - -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ - ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread -fi -SRC=./speedtest1.c -gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread -ls -l speedtest1 | tee -a summary-$NAME.txt -valgrind --tool=cachegrind ./speedtest1 speedtest1.db \ - $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt -size sqlite3.o | tee -a summary-$NAME.txt -wc sqlite3.c -cg_anno.tcl cachegrind.out.* >cout-$NAME.txt -if test $doExplain -eq 1; then - ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt -fi diff --git a/tool/speed-check.sh b/tool/speed-check.sh deleted file mode 100644 index fd122a12db..0000000000 --- a/tool/speed-check.sh +++ /dev/null @@ -1,219 +0,0 @@ -#!/bin/bash -# -# This is a template for a script used for day-to-day size and -# performance monitoring of SQLite. Typical usage: -# -# sh speed-check.sh trunk # Baseline measurement of trunk -# sh speed-check.sh x1 # Measure some experimental change -# fossil xdiff --tk cout-trunk.txt cout-x1.txt # View changes -# -# There are multiple output files, all with a base name given by -# the first argument: -# -# summary-$BASE.txt # Copy of standard output -# cout-$BASE.txt # cachegrind output -# explain-$BASE.txt # EXPLAIN listings (only with --explain) -# -if test "$1" = "" -then - echo "Usage: $0 OUTPUTFILE [OPTIONS]" - exit -fi -NAME=$1 -shift -#CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5" -CC_OPTS="-DSQLITE_ENABLE_MEMSYS5" -CC=gcc -SPEEDTEST_OPTS="--shrink-memory --reprepare --stats --heap 10000000 64" -SIZE=5 -LEAN_OPTS="-DSQLITE_THREADSAFE=0" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_MEMSTATUS=0" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOBS" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_MAX_EXPR_DEPTH=0" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DECLTYPE" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DEPRECATED" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA" -BASELINE="trunk" -doExplain=0 -doCachegrind=1 -doVdbeProfile=0 -doWal=1 -doDiff=1 -while test "$1" != ""; do - case $1 in - --nodiff) - doDiff=0 - ;; - --reprepare) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --autovacuum) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --utf16be) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --stats) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --without-rowid) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --strict) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --nomemstat) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --multithread) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --singlethread) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --serialized) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --temp) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6" - ;; - --legacy) - doWal=0 - CC_OPTS="$CC_OPTS -DSPEEDTEST_OMIT_HASH" - ;; - --verify) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --verify" - ;; - --wal) - doWal=1 - ;; - --size) - shift; SIZE=$1 - ;; - --cachesize) - shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --cachesize $1" - ;; - --stmtcache) - shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtcache $1" - ;; - --checkpoint) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --checkpoint" - ;; - --explain) - doExplain=1 - ;; - --vdbeprofile) - rm -f vdbe_profile.out - CC_OPTS="$CC_OPTS -DVDBE_PROFILE" - doCachegrind=0 - doVdbeProfile=1 - ;; - --lean) - CC_OPTS="$CC_OPTS $LEAN_OPTS" - ;; - --clang) - CC=clang - ;; - --icc) - CC=/home/drh/intel/bin/icc - ;; - --gcc7) - CC=gcc-7 - ;; - --heap) - CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5" - shift; - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64" - ;; - --lookaside) - shift; - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --lookaside $1 $2" - shift; - ;; - --repeat) - CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RCACHE" - shift; - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --repeat $1" - ;; - --mmap) - shift; - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --mmap $1" - ;; - --rtree) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset rtree" - CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RTREE" - ;; - --persist) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --persist" - ;; - --orm) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset orm" - ;; - --cte) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset cte" - ;; - --fp) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp" - ;; - --parsenumber) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset parsenumber" - ;; - --stmtscanstatus) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtscanstatus" - ;; - -*) - CC_OPTS="$CC_OPTS $1" - ;; - *) - BASELINE=$1 - ;; - esac - shift -done -if test $doWal -eq 1; then - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal" -fi -SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE" -echo "NAME = $NAME" | tee summary-$NAME.txt -echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt -echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt -rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o -if test $doVdbeProfile -eq 1; then - rm -f vdbe_profile.out -fi -$CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c -size sqlite3.o | tee -a summary-$NAME.txt -if test $doExplain -eq 1; then - $CC -g -Os -Wall -I. $CC_OPTS \ - -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ - ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread -fi -SRC=./speedtest1.c -$CC -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread -ls -l speedtest1 | tee -a summary-$NAME.txt -if test $doCachegrind -eq 1; then - valgrind --tool=cachegrind ./speedtest1 speedtest1.db \ - $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt -else - ./speedtest1 speedtest1.db $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt -fi -size sqlite3.o | tee -a summary-$NAME.txt -wc sqlite3.c -if test $doCachegrind -eq 1; then - cg_anno.tcl cachegrind.out.* >cout-$NAME.txt - echo '*****************************************************' >>cout-$NAME.txt - sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>cout-$NAME.txt -fi -if test $doExplain -eq 1; then - ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt -fi -if test $doVdbeProfile -eq 1; then - tclsh ../sqlite/tool/vdbe_profile.tcl >vdbeprofile-$NAME.txt - open vdbeprofile-$NAME.txt -fi -if test "$NAME" != "$BASELINE" -a $doVdbeProfile -ne 1 -a $doDiff -ne 0; then - fossil test-diff --tk -c 20 cout-$BASELINE.txt cout-$NAME.txt -fi diff --git a/tool/speedtest.tcl b/tool/speedtest.tcl deleted file mode 100644 index ef39dc5461..0000000000 --- a/tool/speedtest.tcl +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/tclsh -# -# Run this script using TCLSH to do a speed comparison between -# various versions of SQLite and PostgreSQL and MySQL -# - -# Run a test -# -set cnt 1 -proc runtest {title} { - global cnt - set sqlfile test$cnt.sql - puts "

Test $cnt: $title

" - incr cnt - set fd [open $sqlfile r] - set sql [string trim [read $fd [file size $sqlfile]]] - close $fd - set sx [split $sql \n] - set n [llength $sx] - if {$n>8} { - set sql {} - for {set i 0} {$i<3} {incr i} {append sql [lindex $sx $i]
\n} - append sql "... [expr {$n-6}] lines omitted
\n" - for {set i [expr {$n-3}]} {$i<$n} {incr i} { - append sql [lindex $sx $i]
\n - } - } else { - regsub -all \n [string trim $sql]
sql - } - puts "
" - puts "$sql" - puts "
" - set format {} - set delay 1000 -# exec sync; after $delay; -# set t [time "exec psql drh <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format PostgreSQL: $t] - exec sync; after $delay; - set t [time "exec mysql -f drh <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format MySQL: $t] -# set t [time "exec ./sqlite232 s232.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.3.2:} $t] -# set t [time "exec ./sqlite-100 s100.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.4 (cache=100):} $t] - exec sync; after $delay; - set t [time "exec ./sqlite248 s2k.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4.8:} $t] - exec sync; after $delay; - set t [time "exec ./sqlite248 sns.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4.8 (nosync):} $t] - exec sync; after $delay; - set t [time "exec ./sqlite2412 s2kb.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4.12:} $t] - exec sync; after $delay; - set t [time "exec ./sqlite2412 snsb.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4.12 (nosync):} $t] -# set t [time "exec ./sqlite-t1 st1.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.4 (test):} $t] - puts "
%s   %.3f
" -} - -# Initialize the environment -# -expr srand(1) -catch {exec /bin/sh -c {rm -f s*.db}} -set fd [open clear.sql w] -puts $fd { - drop table t1; - drop table t2; -} -close $fd -catch {exec psql drh =1000} { - set txt "[number_name [expr {$n/1000}]] thousand" - set n [expr {$n%1000}] - } else { - set txt {} - } - if {$n>=100} { - append txt " [lindex $::ones [expr {$n/100}]] hundred" - set n [expr {$n%100}] - } - if {$n>=20} { - append txt " [lindex $::tens [expr {$n/10}]]" - set n [expr {$n%10}] - } - if {$n>0} { - append txt " [lindex $::ones $n]" - } - set txt [string trim $txt] - if {$txt==""} {set txt zero} - return $txt -} - - - -set fd [open test$cnt.sql w] -puts $fd "CREATE TABLE t1(a INTEGER, b INTEGER, c VARCHAR(100));" -for {set i 1} {$i<=1000} {incr i} { - set r [expr {int(rand()*100000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -close $fd -runtest {1000 INSERTs} - - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -puts $fd "CREATE TABLE t2(a INTEGER, b INTEGER, c VARCHAR(100));" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t2 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - - -set fd [open test$cnt.sql w] -for {set i 0} {$i<100} {incr i} { - set lwr [expr {$i*100}] - set upr [expr {($i+10)*100}] - puts $fd "SELECT count(*), avg(b) FROM t2 WHERE b>=$lwr AND b<$upr;" -} -close $fd -runtest {100 SELECTs without an index} - - - -set fd [open test$cnt.sql w] -for {set i 1} {$i<=100} {incr i} { - puts $fd "SELECT count(*), avg(b) FROM t2 WHERE c LIKE '%[number_name $i]%';" -} -close $fd -runtest {100 SELECTs on a string comparison} - - - -set fd [open test$cnt.sql w] -puts $fd {CREATE INDEX i2a ON t2(a);} -puts $fd {CREATE INDEX i2b ON t2(b);} -close $fd -runtest {Creating an index} - - - -set fd [open test$cnt.sql w] -for {set i 0} {$i<5000} {incr i} { - set lwr [expr {$i*100}] - set upr [expr {($i+1)*100}] - puts $fd "SELECT count(*), avg(b) FROM t2 WHERE b>=$lwr AND b<$upr;" -} -close $fd -runtest {5000 SELECTs with an index} - - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 0} {$i<1000} {incr i} { - set lwr [expr {$i*10}] - set upr [expr {($i+1)*10}] - puts $fd "UPDATE t1 SET b=b*2 WHERE a>=$lwr AND a<$upr;" -} -puts $fd "COMMIT;" -close $fd -runtest {1000 UPDATEs without an index} - - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "UPDATE t2 SET b=$r WHERE a=$i;" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 UPDATEs with an index} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "UPDATE t2 SET c='[number_name $r]' WHERE a=$i;" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 text UPDATEs with an index} - - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -puts $fd "INSERT INTO t1 SELECT * FROM t2;" -puts $fd "INSERT INTO t2 SELECT * FROM t1;" -puts $fd "COMMIT;" -close $fd -runtest {INSERTs from a SELECT} - - - -set fd [open test$cnt.sql w] -puts $fd {DELETE FROM t2 WHERE c LIKE '%fifty%';} -close $fd -runtest {DELETE without an index} - - - -set fd [open test$cnt.sql w] -puts $fd {DELETE FROM t2 WHERE a>10 AND a<20000;} -close $fd -runtest {DELETE with an index} - - - -set fd [open test$cnt.sql w] -puts $fd {INSERT INTO t2 SELECT * FROM t1;} -close $fd -runtest {A big INSERT after a big DELETE} - - - -set fd [open test$cnt.sql w] -puts $fd {BEGIN;} -puts $fd {DELETE FROM t1;} -for {set i 1} {$i<=3000} {incr i} { - set r [expr {int(rand()*100000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd {COMMIT;} -close $fd -runtest {A big DELETE followed by many small INSERTs} - - - -set fd [open test$cnt.sql w] -puts $fd {DROP TABLE t1;} -puts $fd {DROP TABLE t2;} -close $fd -runtest {DROP TABLE} diff --git a/tool/speedtest16.c b/tool/speedtest16.c deleted file mode 100644 index 993cc19268..0000000000 --- a/tool/speedtest16.c +++ /dev/null @@ -1,171 +0,0 @@ -/* -** Performance test for SQLite. -** -** This program reads ASCII text from a file named on the command-line. -** It converts each SQL statement into UTF16 and submits it to SQLite -** for evaluation. A new UTF16 database is created at the beginning of -** the program. All statements are timed using the high-resolution timer -** built into Intel-class processors. -** -** To compile this program, first compile the SQLite library separately -** will full optimizations. For example: -** -** gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c -** -** Then link against this program. But to do optimize this program -** because that defeats the hi-res timer. -** -** gcc speedtest16.c sqlite3.o -ldl -I../src -** -** Then run this program with a single argument which is the name of -** a file containing SQL script that you want to test: -** -** ./a.out database.db test.sql -*/ -#include -#include -#include -#include -#include -#include "sqlite3.h" - -#define ISSPACE(X) isspace((unsigned char)(X)) - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -#include "hwtime.h" - -/* -** Convert a zero-terminated ASCII string into a zero-terminated -** UTF-16le string. Memory to hold the returned string comes -** from malloc() and should be freed by the caller. -*/ -static void *asciiToUtf16le(const char *z){ - int n = strlen(z); - char *z16; - int i, j; - - z16 = malloc( n*2 + 2 ); - for(i=j=0; i<=n; i++){ - z16[j++] = z[i]; - z16[j++] = 0; - } - return (void*)z16; -} - -/* -** Timers -*/ -static sqlite_uint64 prepTime = 0; -static sqlite_uint64 runTime = 0; -static sqlite_uint64 finalizeTime = 0; - -/* -** Prepare and run a single statement of SQL. -*/ -static void prepareAndRun(sqlite3 *db, const char *zSql){ - void *utf16; - sqlite3_stmt *pStmt; - const void *stmtTail; - sqlite_uint64 iStart, iElapse; - int rc; - - printf("****************************************************************\n"); - printf("SQL statement: [%s]\n", zSql); - utf16 = asciiToUtf16le(zSql); - iStart = sqlite3Hwtime(); - rc = sqlite3_prepare16_v2(db, utf16, -1, &pStmt, &stmtTail); - iElapse = sqlite3Hwtime() - iStart; - prepTime += iElapse; - printf("sqlite3_prepare16_v2() returns %d in %llu cycles\n", rc, iElapse); - if( rc==SQLITE_OK ){ - int nRow = 0; - iStart = sqlite3Hwtime(); - while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; } - iElapse = sqlite3Hwtime() - iStart; - runTime += iElapse; - printf("sqlite3_step() returns %d after %d rows in %llu cycles\n", - rc, nRow, iElapse); - iStart = sqlite3Hwtime(); - rc = sqlite3_finalize(pStmt); - iElapse = sqlite3Hwtime() - iStart; - finalizeTime += iElapse; - printf("sqlite3_finalize() returns %d in %llu cycles\n", rc, iElapse); - } - free(utf16); -} - -int main(int argc, char **argv){ - void *utf16; - sqlite3 *db; - int rc; - int nSql; - char *zSql; - int i, j; - FILE *in; - sqlite_uint64 iStart, iElapse; - sqlite_uint64 iSetup = 0; - int nStmt = 0; - int nByte = 0; - - if( argc!=3 ){ - fprintf(stderr, "Usage: %s FILENAME SQL-SCRIPT\n" - "Runs SQL-SCRIPT as UTF16 against a UTF16 database\n", - argv[0]); - exit(1); - } - in = fopen(argv[2], "r"); - fseek(in, 0L, SEEK_END); - nSql = ftell(in); - zSql = malloc( nSql+1 ); - fseek(in, 0L, SEEK_SET); - nSql = fread(zSql, 1, nSql, in); - zSql[nSql] = 0; - - printf("SQLite version: %d\n", sqlite3_libversion_number()); - unlink(argv[1]); - utf16 = asciiToUtf16le(argv[1]); - iStart = sqlite3Hwtime(); - rc = sqlite3_open16(utf16, &db); - iElapse = sqlite3Hwtime() - iStart; - iSetup = iElapse; - printf("sqlite3_open16() returns %d in %llu cycles\n", rc, iElapse); - free(utf16); - for(i=j=0; jTest $cnt: $title" - incr cnt - set fd [open $sqlfile r] - set sql [string trim [read $fd [file size $sqlfile]]] - close $fd - set sx [split $sql \n] - set n [llength $sx] - if {$n>8} { - set sql {} - for {set i 0} {$i<3} {incr i} {append sql [lindex $sx $i]
\n} - append sql "... [expr {$n-6}] lines omitted
\n" - for {set i [expr {$n-3}]} {$i<$n} {incr i} { - append sql [lindex $sx $i]
\n - } - } else { - regsub -all \n [string trim $sql]
sql - } - puts "
" - puts "$sql" - puts "
" - set format {} - set delay 1000 - exec sync; after $delay; - set t [time "exec psql drh <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format PostgreSQL: $t] - exec sync; after $delay; - set t [time "exec mysql -f drh <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format MySQL: $t] -# set t [time "exec ./sqlite232 s232.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.3.2:} $t] -# set t [time "exec ./sqlite-100 s100.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.4 (cache=100):} $t] - exec sync; after $delay; - set t [time "exec ./sqlite240 s2k.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4:} $t] - exec sync; after $delay; - set t [time "exec ./sqlite240 sns.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4 (nosync):} $t] -# set t [time "exec ./sqlite-t1 st1.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.4 (test):} $t] - puts "
%s   %.3f
" -} - -# Initialize the environment -# -expr srand(1) -catch {exec /bin/sh -c {rm -f s*.db}} -set fd [open clear.sql w] -puts $fd { - drop table t1; - drop table t2; -} -close $fd -catch {exec psql drh =1000} { - set txt "[number_name [expr {$n/1000}]] thousand" - set n [expr {$n%1000}] - } else { - set txt {} - } - if {$n>=100} { - append txt " [lindex $::ones [expr {$n/100}]] hundred" - set n [expr {$n%100}] - } - if {$n>=20} { - append txt " [lindex $::tens [expr {$n/10}]]" - set n [expr {$n%10}] - } - if {$n>0} { - append txt " [lindex $::ones $n]" - } - set txt [string trim $txt] - if {$txt==""} {set txt zero} - return $txt -} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -puts $fd "CREATE TABLE t1(a INTEGER, b INTEGER, c VARCHAR(100));" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - -set fd [open test$cnt.sql w] -puts $fd "DELETE FROM t1;" -close $fd -runtest {DELETE everything} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - -set fd [open test$cnt.sql w] -puts $fd "DELETE FROM t1;" -close $fd -runtest {DELETE everything} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - -set fd [open test$cnt.sql w] -puts $fd "DELETE FROM t1;" -close $fd -runtest {DELETE everything} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - -set fd [open test$cnt.sql w] -puts $fd "DELETE FROM t1;" -close $fd -runtest {DELETE everything} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - -set fd [open test$cnt.sql w] -puts $fd "DELETE FROM t1;" -close $fd -runtest {DELETE everything} - - -set fd [open test$cnt.sql w] -puts $fd {DROP TABLE t1;} -close $fd -runtest {DROP TABLE} diff --git a/tool/speedtest8.c b/tool/speedtest8.c deleted file mode 100644 index 051fc89819..0000000000 --- a/tool/speedtest8.c +++ /dev/null @@ -1,260 +0,0 @@ -/* -** Performance test for SQLite. -** -** This program reads ASCII text from a file named on the command-line -** and submits that text to SQLite for evaluation. A new database -** is created at the beginning of the program. All statements are -** timed using the high-resolution timer built into Intel-class processors. -** -** To compile this program, first compile the SQLite library separately -** will full optimizations. For example: -** -** gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c -** -** Then link against this program. But to do optimize this program -** because that defeats the hi-res timer. -** -** gcc speedtest8.c sqlite3.o -ldl -I../src -** -** Then run this program with a single argument which is the name of -** a file containing SQL script that you want to test: -** -** ./a.out test.db test.sql -*/ -#include -#include -#include -#include -#include - -#if defined(_MSC_VER) -#include -#else -#include -#include -#include -#endif - -#include "sqlite3.h" - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -#include "hwtime.h" - -/* -** Timers -*/ -static sqlite_uint64 prepTime = 0; -static sqlite_uint64 runTime = 0; -static sqlite_uint64 finalizeTime = 0; - -/* -** Prepare and run a single statement of SQL. -*/ -static void prepareAndRun(sqlite3 *db, const char *zSql, int bQuiet){ - sqlite3_stmt *pStmt; - const char *stmtTail; - sqlite_uint64 iStart, iElapse; - int rc; - - if (!bQuiet){ - printf("***************************************************************\n"); - } - if (!bQuiet) printf("SQL statement: [%s]\n", zSql); - iStart = sqlite3Hwtime(); - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &stmtTail); - iElapse = sqlite3Hwtime() - iStart; - prepTime += iElapse; - if (!bQuiet){ - printf("sqlite3_prepare_v2() returns %d in %llu cycles\n", rc, iElapse); - } - if( rc==SQLITE_OK ){ - int nRow = 0; - iStart = sqlite3Hwtime(); - while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; } - iElapse = sqlite3Hwtime() - iStart; - runTime += iElapse; - if (!bQuiet){ - printf("sqlite3_step() returns %d after %d rows in %llu cycles\n", - rc, nRow, iElapse); - } - iStart = sqlite3Hwtime(); - rc = sqlite3_finalize(pStmt); - iElapse = sqlite3Hwtime() - iStart; - finalizeTime += iElapse; - if (!bQuiet){ - printf("sqlite3_finalize() returns %d in %llu cycles\n", rc, iElapse); - } - } -} - -int main(int argc, char **argv){ - sqlite3 *db; - int rc; - int nSql; - char *zSql; - int i, j; - FILE *in; - sqlite_uint64 iStart, iElapse; - sqlite_uint64 iSetup = 0; - int nStmt = 0; - int nByte = 0; - const char *zArgv0 = argv[0]; - int bQuiet = 0; -#if !defined(_MSC_VER) - struct tms tmsStart, tmsEnd; - clock_t clkStart, clkEnd; -#endif - -#ifdef HAVE_OSINST - extern sqlite3_vfs *sqlite3_instvfs_binarylog(char *, char *, char *); - extern void sqlite3_instvfs_destroy(sqlite3_vfs *); - sqlite3_vfs *pVfs = 0; -#endif - - while (argc>3) - { -#ifdef HAVE_OSINST - if( argc>4 && (strcmp(argv[1], "-log")==0) ){ - pVfs = sqlite3_instvfs_binarylog("oslog", 0, argv[2]); - sqlite3_vfs_register(pVfs, 1); - argv += 2; - argc -= 2; - continue; - } -#endif - - /* - ** Increasing the priority slightly above normal can help with - ** repeatability of testing. Note that with Cygwin, -5 equates - ** to "High", +5 equates to "Low", and anything in between - ** equates to "Normal". - */ - if( argc>4 && (strcmp(argv[1], "-priority")==0) ){ -#if defined(_MSC_VER) - int new_priority = atoi(argv[2]); - if(!SetPriorityClass(GetCurrentProcess(), - (new_priority<=-5) ? HIGH_PRIORITY_CLASS : - (new_priority<=0) ? ABOVE_NORMAL_PRIORITY_CLASS : - (new_priority==0) ? NORMAL_PRIORITY_CLASS : - (new_priority<5) ? BELOW_NORMAL_PRIORITY_CLASS : - IDLE_PRIORITY_CLASS)){ - printf ("error setting priority\n"); - exit(2); - } -#else - struct sched_param myParam; - sched_getparam(0, &myParam); - printf ("Current process priority is %d.\n", (int)myParam.sched_priority); - myParam.sched_priority = atoi(argv[2]); - printf ("Setting process priority to %d.\n", (int)myParam.sched_priority); - if (sched_setparam (0, &myParam) != 0){ - printf ("error setting priority\n"); - exit(2); - } -#endif - argv += 2; - argc -= 2; - continue; - } - - if( argc>3 && strcmp(argv[1], "-quiet")==0 ){ - bQuiet = -1; - argv++; - argc--; - continue; - } - - break; - } - - if( argc!=3 ){ - fprintf(stderr, "Usage: %s [options] FILENAME SQL-SCRIPT\n" - "Runs SQL-SCRIPT against a UTF8 database\n" - "\toptions:\n" -#ifdef HAVE_OSINST - "\t-log \n" -#endif - "\t-priority : set priority of task\n" - "\t-quiet : only display summary results\n", - zArgv0); - exit(1); - } - - in = fopen(argv[2], "r"); - fseek(in, 0L, SEEK_END); - nSql = ftell(in); - zSql = malloc( nSql+1 ); - fseek(in, 0L, SEEK_SET); - nSql = fread(zSql, 1, nSql, in); - zSql[nSql] = 0; - - printf("SQLite version: %d\n", sqlite3_libversion_number()); - unlink(argv[1]); -#if !defined(_MSC_VER) - clkStart = times(&tmsStart); -#endif - iStart = sqlite3Hwtime(); - rc = sqlite3_open(argv[1], &db); - iElapse = sqlite3Hwtime() - iStart; - iSetup = iElapse; - if (!bQuiet) printf("sqlite3_open() returns %d in %llu cycles\n", rc, iElapse); - for(i=j=0; j=6 && memcmp(&zSql[i], ".crash",6)==0 ) exit(1); - nStmt++; - nByte += n; - prepareAndRun(db, &zSql[i], bQuiet); - } - zSql[j] = ';'; - i = j+1; - } - } - } - iStart = sqlite3Hwtime(); - sqlite3_close(db); - iElapse = sqlite3Hwtime() - iStart; -#if !defined(_MSC_VER) - clkEnd = times(&tmsEnd); -#endif - iSetup += iElapse; - if (!bQuiet) printf("sqlite3_close() returns in %llu cycles\n", iElapse); - - printf("\n"); - printf("Statements run: %15d stmts\n", nStmt); - printf("Bytes of SQL text: %15d bytes\n", nByte); - printf("Total prepare time: %15llu cycles\n", prepTime); - printf("Total run time: %15llu cycles\n", runTime); - printf("Total finalize time: %15llu cycles\n", finalizeTime); - printf("Open/Close time: %15llu cycles\n", iSetup); - printf("Total time: %15llu cycles\n", - prepTime + runTime + finalizeTime + iSetup); - -#if !defined(_MSC_VER) - printf("\n"); - printf("Total user CPU time: %15.3g secs\n", (tmsEnd.tms_utime - tmsStart.tms_utime)/(double)CLOCKS_PER_SEC ); - printf("Total system CPU time: %15.3g secs\n", (tmsEnd.tms_stime - tmsStart.tms_stime)/(double)CLOCKS_PER_SEC ); - printf("Total real time: %15.3g secs\n", (clkEnd -clkStart)/(double)CLOCKS_PER_SEC ); -#endif - -#ifdef HAVE_OSINST - if( pVfs ){ - sqlite3_instvfs_destroy(pVfs); - printf("vfs log written to %s\n", argv[0]); - } -#endif - - return 0; -} diff --git a/tool/speedtest8inst1.c b/tool/speedtest8inst1.c deleted file mode 100644 index ceaeca0f16..0000000000 --- a/tool/speedtest8inst1.c +++ /dev/null @@ -1,218 +0,0 @@ -/* -** Performance test for SQLite. -** -** This program reads ASCII text from a file named on the command-line -** and submits that text to SQLite for evaluation. A new database -** is created at the beginning of the program. All statements are -** timed using the high-resolution timer built into Intel-class processors. -** -** To compile this program, first compile the SQLite library separately -** will full optimizations. For example: -** -** gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c -** -** Then link against this program. But to do optimize this program -** because that defeats the hi-res timer. -** -** gcc speedtest8.c sqlite3.o -ldl -I../src -** -** Then run this program with a single argument which is the name of -** a file containing SQL script that you want to test: -** -** ./a.out test.db test.sql -*/ -#include -#include -#include -#include -#include -#include -#include "sqlite3.h" - -#define ISSPACE(X) isspace((unsigned char)(X)) - -#include "test_osinst.c" - -/* -** Prepare and run a single statement of SQL. -*/ -static void prepareAndRun(sqlite3_vfs *pInstVfs, sqlite3 *db, const char *zSql){ - sqlite3_stmt *pStmt; - const char *stmtTail; - int rc; - char zMessage[1024]; - zMessage[1023] = '\0'; - - sqlite3_uint64 iTime; - - sqlite3_snprintf(1023, zMessage, "sqlite3_prepare_v2: %s", zSql); - sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage); - - iTime = sqlite3Hwtime(); - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &stmtTail); - iTime = sqlite3Hwtime() - iTime; - sqlite3_instvfs_binarylog_call(pInstVfs,BINARYLOG_PREPARE_V2,iTime,rc,zSql); - - if( rc==SQLITE_OK ){ - int nRow = 0; - - sqlite3_snprintf(1023, zMessage, "sqlite3_step loop: %s", zSql); - sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage); - iTime = sqlite3Hwtime(); - while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; } - iTime = sqlite3Hwtime() - iTime; - sqlite3_instvfs_binarylog_call(pInstVfs, BINARYLOG_STEP, iTime, rc, zSql); - - sqlite3_snprintf(1023, zMessage, "sqlite3_finalize: %s", zSql); - sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage); - iTime = sqlite3Hwtime(); - rc = sqlite3_finalize(pStmt); - iTime = sqlite3Hwtime() - iTime; - sqlite3_instvfs_binarylog_call(pInstVfs, BINARYLOG_FINALIZE, iTime, rc, zSql); - } -} - -static int stringcompare(const char *zLeft, const char *zRight){ - int ii; - for(ii=0; zLeft[ii] && zRight[ii]; ii++){ - if( zLeft[ii]!=zRight[ii] ) return 0; - } - return( zLeft[ii]==zRight[ii] ); -} - -static char *readScriptFile(const char *zFile, int *pnScript){ - sqlite3_vfs *pVfs = sqlite3_vfs_find(0); - sqlite3_file *p; - int rc; - sqlite3_int64 nByte; - char *zData = 0; - int flags = SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_DB; - - p = (sqlite3_file *)malloc(pVfs->szOsFile); - rc = pVfs->xOpen(pVfs, zFile, p, flags, &flags); - if( rc!=SQLITE_OK ){ - goto error_out; - } - - rc = p->pMethods->xFileSize(p, &nByte); - if( rc!=SQLITE_OK ){ - goto close_out; - } - - zData = (char *)malloc(nByte+1); - rc = p->pMethods->xRead(p, zData, nByte, 0); - if( rc!=SQLITE_OK ){ - goto close_out; - } - zData[nByte] = '\0'; - - p->pMethods->xClose(p); - free(p); - *pnScript = nByte; - return zData; - -close_out: - p->pMethods->xClose(p); - -error_out: - free(p); - free(zData); - return 0; -} - -int main(int argc, char **argv){ - - const char zUsageMsg[] = - "Usage: %s options...\n" - " where available options are:\n" - "\n" - " -db DATABASE-FILE (database file to operate on)\n" - " -script SCRIPT-FILE (script file to read sql from)\n" - " -log LOG-FILE (log file to create)\n" - " -logdata (log all data to log file)\n" - "\n" - " Options -db, -script and -log are compulsory\n" - "\n" - ; - - const char *zDb = 0; - const char *zScript = 0; - const char *zLog = 0; - int logdata = 0; - - int ii; - int i, j; - int rc; - - sqlite3_vfs *pInstVfs; /* Instrumentation VFS */ - - char *zSql = 0; - int nSql; - - sqlite3 *db; - - for(ii=1; ii Date: Mon, 28 Apr 2025 15:00:28 +0000 Subject: [PATCH 14/42] Remove a stray blank line from the test/testrunner.tcl script. FossilOrigin-Name: 7d6005b0d17d86d72757f3578872e82fd12f61ac9160a0277e397df09f6338a9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 990637bbe5..9fd5d914af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\slong-obsolete\sperformance\stesting\sscripts\sfrom\sthe\stools/\sfolder.\nThe\stest/speedtest.tcl\sscript\sis\swhat\sshould\sbe\sused\sthese\sdays.\s\sThis\nis\sjust\scruft\sclean-up. -D 2025-04-28T13:32:46.678 +C Remove\sa\sstray\sblank\sline\sfrom\sthe\stest/testrunner.tcl\sscript. +D 2025-04-28T15:00:28.623 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -1734,7 +1734,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl 228732ca62ea60e97785e13528bd36627cf3c115d59364a5181ed7f85fd0031d x +F test/testrunner.tcl de6fe8e645b864b49bd20d2dd968cb5863313ff5138ba7e22841d99e8054cf56 x F test/testrunner_data.tcl 07f2d36a6321bfd196975aca237db6bba1991032809264a1abeb62e697196d20 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0ea5993dd8549056d73f3d5447e0a810495ece23a75be8add78fba6842125d85 -R e9a7a5bb7d30c86d12fb872b44065ee2 +P 956af8c294ae5b826a5b645922df07dad360290d19bf2b6a7f74d0a783a3e692 +R 13056561c2ba68bd2352898ef95bb642 U drh -Z faa21c2c7bc49d38601dda410afe3656 +Z 8a151df5f2e0c562bd71995175f23f56 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a89eb6ee7d..425e5b40bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -956af8c294ae5b826a5b645922df07dad360290d19bf2b6a7f74d0a783a3e692 +7d6005b0d17d86d72757f3578872e82fd12f61ac9160a0277e397df09f6338a9 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 60c4627f92..051f15d16c 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -169,7 +169,6 @@ Full documentation here: https://sqlite.org/src/doc/trunk/doc/testrunner.md proc guess_number_of_cores {} { if {[catch {number_of_cores} ret]} { set ret 4 - if {$::tcl_platform(platform) eq "windows"} { catch { set ret $::env(NUMBER_OF_PROCESSORS) } } else { From 7fec7c96c478848a769d9c35ecd2e00edb31612d Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Apr 2025 16:17:17 +0000 Subject: [PATCH 15/42] Fix testrunner.tcl so that it honors pattern matches in non-testfixture tests, such as fuzzcheck and sessionfuzz. FossilOrigin-Name: 81857474fe90f50b760c22d61a6ad477642dabe1052390cd1614dcec8adb1376 --- manifest | 12 +++---- manifest.uuid | 2 +- test/testrunner.tcl | 80 +++++++++++++++++++++++++-------------------- 3 files changed, 52 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 9fd5d914af..cd36ef9f65 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sstray\sblank\sline\sfrom\sthe\stest/testrunner.tcl\sscript. -D 2025-04-28T15:00:28.623 +C Fix\stestrunner.tcl\sso\sthat\sit\shonors\spattern\smatches\sin\snon-testfixture\ntests,\ssuch\sas\sfuzzcheck\sand\ssessionfuzz. +D 2025-04-28T16:17:17.167 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -1734,7 +1734,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl de6fe8e645b864b49bd20d2dd968cb5863313ff5138ba7e22841d99e8054cf56 x +F test/testrunner.tcl 9cf2bd5755077b710b4675ddd20e88271a6baaf8d94c19dbb7021798cec7d537 x F test/testrunner_data.tcl 07f2d36a6321bfd196975aca237db6bba1991032809264a1abeb62e697196d20 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 956af8c294ae5b826a5b645922df07dad360290d19bf2b6a7f74d0a783a3e692 -R 13056561c2ba68bd2352898ef95bb642 +P 7d6005b0d17d86d72757f3578872e82fd12f61ac9160a0277e397df09f6338a9 +R 65434eb108e608265878d208220c17e3 U drh -Z 8a151df5f2e0c562bd71995175f23f56 +Z 8c6e3b2216a6b6e244ba52b67889d195 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 425e5b40bf..ddf5a4051d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7d6005b0d17d86d72757f3578872e82fd12f61ac9160a0277e397df09f6338a9 +81857474fe90f50b760c22d61a6ad477642dabe1052390cd1614dcec8adb1376 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 051f15d16c..479f17868c 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1002,6 +1002,35 @@ proc add_job {args} { trdb last_insert_rowid } + +# Look to see if $jobcmd matches any of the glob patterns given in +# $patternlist. Return true if there is a match. Return false +# if no match is seen. +# +# An empty patternlist matches everything +# +proc job_matches_any_pattern {patternlist jobcmd} { + set bMatch 0 + if {[llength $patternlist]==0} {return 1} + foreach p $patternlist { + set p [string trim $p *] + if {[string index $p 0]=="^"} { + set p [string range $p 1 end] + } else { + set p "*$p" + } + if {[string index $p end]=="\$"} { + set p [string range $p 0 end-1] + } else { + set p "$p*" + } + if {[string match $p $jobcmd]} { + set bMatch 1 + break + } + } + return $bMatch +} # Argument $build is either an empty string, or else a list of length 3 @@ -1032,26 +1061,8 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { # The ::testspec array is populated by permutations.test foreach f [dict get $::testspec($config) -files] { - if {[llength $patternlist]>0} { - set bMatch 0 - foreach p $patternlist { - set p [string trim $p *] - if {[string index $p 0]=="^"} { - set p [string range $p 1 end] - } else { - set p "*$p" - } - if {[string index $p end]=="\$"} { - set p [string range $p 0 end-1] - } else { - set p "$p*" - } - if {[string match $p "$config [file tail $f]"]} { - set bMatch 1 - break - } - } - if {$bMatch==0} continue + if {![job_matches_any_pattern $patternlist "$config [file tail $f]"]} { + continue } if {[file pathtype $f]!="absolute"} { set f [file join $::testdir $f] } @@ -1145,26 +1156,29 @@ proc add_make_job {bld target} { -priority 1 } -proc add_fuzztest_jobs {buildname} { +proc add_fuzztest_jobs {buildname patternlist} { foreach {interpreter scripts} [trd_fuzztest_data] { + set bldDone 0 set subcmd [lrange $interpreter 1 end] set interpreter [lindex $interpreter 0] - set bld [add_build_job $buildname $interpreter] - foreach {depid dirname displayname} $bld {} - foreach s $scripts { # Fuzz data files fuzzdata1.db and fuzzdata2.db are larger than # the others. So ensure that these are run as a higher priority. set tail [file tail $s] + if {![job_matches_any_pattern $patternlist "fuzzcheck $tail"]} continue + if {!$bldDone} { + set bld [add_build_job $buildname $interpreter] + foreach {depid dirname displayname} $bld {} + set bldDone 1 + } if {$tail=="fuzzdata1.db" || $tail=="fuzzdata2.db"} { set priority 5 } else { set priority 1 } - add_job \ -displaytype fuzz \ -displayname "$buildname $interpreter $tail" \ @@ -1200,9 +1214,7 @@ proc add_devtest_jobs {lBld patternlist} { foreach b $lBld { set bld [add_build_job $b $TRG(testfixture)] add_tcl_jobs $bld veryquick $patternlist SHELL - if {$patternlist==""} { - add_fuzztest_jobs $b - } + add_fuzztest_jobs $b $patternlist if {[trdb one "SELECT EXISTS (SELECT 1 FROM jobs WHERE depid='SHELL')"]} { set sbld [add_shell_build_job $b [lindex $bld 1] [lindex $bld 0]] @@ -1276,13 +1288,11 @@ proc add_jobs_from_cmdline {patternlist} { add_tcl_jobs $bld $c $patternlist SHELL } - if {$patternlist==""} { - foreach e [trd_extras $TRG(platform) $b] { - if {$e=="fuzztest"} { - add_fuzztest_jobs $b - } else { - add_make_job $bld $e - } + foreach e [trd_extras $TRG(platform) $b] { + if {$e=="fuzztest"} { + add_fuzztest_jobs $b $patternlist + } elseif {[job_matches_any_pattern $patternlist $e]} { + add_make_job $bld $e } } From 1365bcffbf871273ace51461ad46826ebc88e30b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Apr 2025 16:28:00 +0000 Subject: [PATCH 16/42] Enhance testrunner.tcl so that it does not build testfixture unnecessarily - if the pattern does not match any TCL test. FossilOrigin-Name: 896d767f1cba64bda7b91b0daebd60ebe4457a58d762da443bf61cf5b39ad2d9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index cd36ef9f65..def9eb2d88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stestrunner.tcl\sso\sthat\sit\shonors\spattern\smatches\sin\snon-testfixture\ntests,\ssuch\sas\sfuzzcheck\sand\ssessionfuzz. -D 2025-04-28T16:17:17.167 +C Enhance\stestrunner.tcl\sso\sthat\sit\sdoes\snot\sbuild\stestfixture\sunnecessarily\s-\nif\sthe\spattern\sdoes\snot\smatch\sany\sTCL\stest. +D 2025-04-28T16:28:00.311 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -1734,7 +1734,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl 9cf2bd5755077b710b4675ddd20e88271a6baaf8d94c19dbb7021798cec7d537 x +F test/testrunner.tcl 9ba1633c0494fde4bc5a15dad7393c96cab5842674d45ae3fedbb73e646e4a8a x F test/testrunner_data.tcl 07f2d36a6321bfd196975aca237db6bba1991032809264a1abeb62e697196d20 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7d6005b0d17d86d72757f3578872e82fd12f61ac9160a0277e397df09f6338a9 -R 65434eb108e608265878d208220c17e3 +P 81857474fe90f50b760c22d61a6ad477642dabe1052390cd1614dcec8adb1376 +R 2a7e3de3ac92b862d831bf5772b280f3 U drh -Z 8c6e3b2216a6b6e244ba52b67889d195 +Z aaa50756b963ce47265747d8b22c07a0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ddf5a4051d..159a066853 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -81857474fe90f50b760c22d61a6ad477642dabe1052390cd1614dcec8adb1376 +896d767f1cba64bda7b91b0daebd60ebe4457a58d762da443bf61cf5b39ad2d9 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 479f17868c..480d70a910 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1044,6 +1044,7 @@ proc job_matches_any_pattern {patternlist jobcmd} { # proc add_tcl_jobs {build config patternlist {shelldepid ""}} { global TRG + set ntcljob 0 set topdir [file dirname $::testdir] set testrunner_tcl [file normalize [info script]] @@ -1087,6 +1088,7 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { set depid [lindex $build 0] if {$shelldepid!="" && [lsearch $lProp shell]>=0} { set depid $shelldepid } + incr ntcljob add_job \ -displaytype tcl \ -displayname $displayname \ @@ -1094,6 +1096,10 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { -depid $depid \ -priority $priority } + if {$ntcljob==0 && [llength $build]>0} { + set bldid [lindex $build 0] + trdb eval {DELETE FROM jobs WHERE rowid=$bldid} + } } proc add_build_job {buildname target {postcmd ""} {depid ""}} { From d4cc3a42e2fe52a0e40eaa9291d91ba4d971e3c3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Apr 2025 17:56:23 +0000 Subject: [PATCH 17/42] If the FUZZDB environment variable is set when running testrunner.tcl and that env-var points to a fuzzcheck database, then suppliment the fuzzcheck tests with that extra database. FossilOrigin-Name: c08530dd6a71cb547d164e2313657c917f280e7988211bf199c8776e78a4e2a6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 30 ++++++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index def9eb2d88..d67ae79daa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\stestrunner.tcl\sso\sthat\sit\sdoes\snot\sbuild\stestfixture\sunnecessarily\s-\nif\sthe\spattern\sdoes\snot\smatch\sany\sTCL\stest. -D 2025-04-28T16:28:00.311 +C If\sthe\sFUZZDB\senvironment\svariable\sis\sset\swhen\srunning\stestrunner.tcl\sand\nthat\senv-var\spoints\sto\sa\sfuzzcheck\sdatabase,\sthen\ssuppliment\sthe\sfuzzcheck\ntests\swith\sthat\sextra\sdatabase. +D 2025-04-28T17:56:23.974 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -1734,7 +1734,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl 9ba1633c0494fde4bc5a15dad7393c96cab5842674d45ae3fedbb73e646e4a8a x +F test/testrunner.tcl 7efcd416e2686fc16a479bc7231204ce4b1473e703136cbcbab2091675b4787c x F test/testrunner_data.tcl 07f2d36a6321bfd196975aca237db6bba1991032809264a1abeb62e697196d20 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 81857474fe90f50b760c22d61a6ad477642dabe1052390cd1614dcec8adb1376 -R 2a7e3de3ac92b862d831bf5772b280f3 +P 896d767f1cba64bda7b91b0daebd60ebe4457a58d762da443bf61cf5b39ad2d9 +R f8483dc6f5ebfe6700dbaa0ae0281899 U drh -Z aaa50756b963ce47265747d8b22c07a0 +Z b87e67a4c1863da23f9d9a564632116e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 159a066853..4b60e64c3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -896d767f1cba64bda7b91b0daebd60ebe4457a58d762da443bf61cf5b39ad2d9 +c08530dd6a71cb547d164e2313657c917f280e7988211bf199c8776e78a4e2a6 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 480d70a910..9b0e5ca4b1 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1163,24 +1163,47 @@ proc add_make_job {bld target} { } proc add_fuzztest_jobs {buildname patternlist} { + global env TRG foreach {interpreter scripts} [trd_fuzztest_data] { set bldDone 0 set subcmd [lrange $interpreter 1 end] set interpreter [lindex $interpreter 0] + if {$interpreter=="fuzzcheck" + && [info exists env(FUZZDB)] + && [file readable $env(FUZZDB)] + } { + set TRG(FUZZDB) $env(FUZZDB) + set fname [file normalize $env(FUZZDB)] + set N [expr {([file size $fname]+4999999)/5000000}] + for {set i 0} {$i<$N} {incr i} { + lappend scripts [list --slice $i $N $fname] + } + } + foreach s $scripts { # Fuzz data files fuzzdata1.db and fuzzdata2.db are larger than # the others. So ensure that these are run as a higher priority. - set tail [file tail $s] + if {[llength $s]==1} { + set tail [file tail $s] + } else { + set fname [lindex $s end] + set tail [lrange $s 0 end-1] + lappend tail [file tail $fname] + } if {![job_matches_any_pattern $patternlist "fuzzcheck $tail"]} continue if {!$bldDone} { set bld [add_build_job $buildname $interpreter] foreach {depid dirname displayname} $bld {} set bldDone 1 } - if {$tail=="fuzzdata1.db" || $tail=="fuzzdata2.db"} { + if {[string match ?-slice* $tail]} { + set priority 15 + } elseif {$tail=="fuzzdata1.db" + || $tail=="fuzzdata2.db" + || $tail=="fuzzdata8.db"} { set priority 5 } else { set priority 1 @@ -1618,6 +1641,9 @@ proc run_testset {} { puts "\nTest database is $TRG(dbname)" puts "Test log is $TRG(logname)" + if {[info exists TRG(FUZZDB)]} { + puts "Extra fuzztest data taken from $TRG(FUZZDB)" + } trdb eval { SELECT sum(ntest) AS totaltest, sum(nerr) AS totalerr From 246dc9235432e83fdf6352d9aba86ea48203f1fa Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 28 Apr 2025 19:15:46 +0000 Subject: [PATCH 18/42] Testrunner.tcl now runs fuzzcheck-asan and fuzzcheck-ubsan tests. FossilOrigin-Name: 3d8f3a255843c063157265702b76d40c38c774d29d658eea2d77efac92ac8789 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/testrunner.tcl | 10 ++++++---- test/testrunner_data.tcl | 5 ++++- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index d67ae79daa..9fc92ec788 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\sFUZZDB\senvironment\svariable\sis\sset\swhen\srunning\stestrunner.tcl\sand\nthat\senv-var\spoints\sto\sa\sfuzzcheck\sdatabase,\sthen\ssuppliment\sthe\sfuzzcheck\ntests\swith\sthat\sextra\sdatabase. -D 2025-04-28T17:56:23.974 +C Testrunner.tcl\snow\sruns\sfuzzcheck-asan\sand\sfuzzcheck-ubsan\stests. +D 2025-04-28T19:15:46.161 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -1734,8 +1734,8 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl 7efcd416e2686fc16a479bc7231204ce4b1473e703136cbcbab2091675b4787c x -F test/testrunner_data.tcl 07f2d36a6321bfd196975aca237db6bba1991032809264a1abeb62e697196d20 +F test/testrunner.tcl c8770b2dea36a6d082f68fda2e1aa21e1c0f893d0ccb326881d82771aa394dad x +F test/testrunner_data.tcl 5c0a0df38586730be14266deb06d122ffb13d3d329f9aeedf53edd7a40b43bbe F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 896d767f1cba64bda7b91b0daebd60ebe4457a58d762da443bf61cf5b39ad2d9 -R f8483dc6f5ebfe6700dbaa0ae0281899 +P c08530dd6a71cb547d164e2313657c917f280e7988211bf199c8776e78a4e2a6 +R ff31b0082651bc2cb2f7c326dab44dfd U drh -Z b87e67a4c1863da23f9d9a564632116e +Z 9ad34f9de95969e24cbdecdb43b8f051 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4b60e64c3f..9a89bd5673 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c08530dd6a71cb547d164e2313657c917f280e7988211bf199c8776e78a4e2a6 +3d8f3a255843c063157265702b76d40c38c774d29d658eea2d77efac92ac8789 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 9b0e5ca4b1..e755cc3840 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -531,7 +531,7 @@ proc show_status {db cls} { (SELECT value FROM config WHERE name='start') }] - set total 0 + set totalw 0 foreach s {"" ready running done failed omit} { set S($s) 0; set W($s) 0; } set workpending 0 $db eval { @@ -556,7 +556,7 @@ proc show_status {db cls} { flush stdout } puts [format %-79.79s "Command: \[testrunner.tcl$cmdline\]"] - puts [format %-79.79s "Summary: [elapsetime $tm], $fin/$total jobs,\ + puts [format %-79.79s "Summary: [elapsetime $tm], $fin/$totalw jobs,\ $ne errors, $nt tests"] set srcdir [file dirname [file dirname $TRG(info_script)]] @@ -1170,7 +1170,7 @@ proc add_fuzztest_jobs {buildname patternlist} { set subcmd [lrange $interpreter 1 end] set interpreter [lindex $interpreter 0] - if {$interpreter=="fuzzcheck" + if {[string match fuzzcheck* $interpreter] && [info exists env(FUZZDB)] && [file readable $env(FUZZDB)] } { @@ -1193,7 +1193,9 @@ proc add_fuzztest_jobs {buildname patternlist} { set tail [lrange $s 0 end-1] lappend tail [file tail $fname] } - if {![job_matches_any_pattern $patternlist "fuzzcheck $tail"]} continue + if {![job_matches_any_pattern $patternlist "$interpreter $tail"]} { + continue + } if {!$bldDone} { set bld [add_build_job $buildname $interpreter] foreach {depid dirname displayname} $bld {} diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index ade126a64d..531eaaff7b 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -459,7 +459,10 @@ proc trd_fuzztest_data {} { return [list fuzzcheck.exe $lFuzzDb] } - return [list [trd_get_bin_name fuzzcheck] $lFuzzDb {sessionfuzz run} $lSessionDb] + return [list [trd_get_bin_name fuzzcheck] $lFuzzDb \ + [trd_get_bin_name fuzzcheck-asan] $lFuzzDb \ + [trd_get_bin_name fuzzcheck-ubsan] $lFuzzDb \ + {sessionfuzz run} $lSessionDb] } From 8a68ab963eacddc7f70754217f3aed8ecfaecf5b Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 29 Apr 2025 01:37:31 +0000 Subject: [PATCH 19/42] Make testrunner.tcl more restrictive about which configurations run fuzzcheck-asan and fuzzcheck-ubsan. FossilOrigin-Name: 785599394dde0fb36ae2e07d611f9523a05a0817dbd8c416582b3b682dbfa4ff --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/testrunner.tcl | 3 ++- test/testrunner_data.tcl | 18 ++++++++++++------ 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 9fc92ec788..19f9e94aca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Testrunner.tcl\snow\sruns\sfuzzcheck-asan\sand\sfuzzcheck-ubsan\stests. -D 2025-04-28T19:15:46.161 +C Make\stestrunner.tcl\smore\srestrictive\sabout\swhich\sconfigurations\srun\nfuzzcheck-asan\sand\sfuzzcheck-ubsan. +D 2025-04-29T01:37:31.258 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -1734,8 +1734,8 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl c8770b2dea36a6d082f68fda2e1aa21e1c0f893d0ccb326881d82771aa394dad x -F test/testrunner_data.tcl 5c0a0df38586730be14266deb06d122ffb13d3d329f9aeedf53edd7a40b43bbe +F test/testrunner.tcl 3ee9722ea8024ad724cd3436ec7661a25bae716b3659a7f0f689c98dc1f40af0 x +F test/testrunner_data.tcl 81bee1515dde9f032c8e0169492a0428b6476dd7cb54276c237938e110db70d5 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c08530dd6a71cb547d164e2313657c917f280e7988211bf199c8776e78a4e2a6 -R ff31b0082651bc2cb2f7c326dab44dfd +P 3d8f3a255843c063157265702b76d40c38c774d29d658eea2d77efac92ac8789 +R f149be995be74f98448d3cbd3e7855b4 U drh -Z 9ad34f9de95969e24cbdecdb43b8f051 +Z 469b94e6776e53378cf6fa455ca8fb8e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9a89bd5673..0c8534eee2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3d8f3a255843c063157265702b76d40c38c774d29d658eea2d77efac92ac8789 +785599394dde0fb36ae2e07d611f9523a05a0817dbd8c416582b3b682dbfa4ff diff --git a/test/testrunner.tcl b/test/testrunner.tcl index e755cc3840..68b7d99740 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1164,8 +1164,9 @@ proc add_make_job {bld target} { proc add_fuzztest_jobs {buildname patternlist} { global env TRG + # puts buildname=$buildname - foreach {interpreter scripts} [trd_fuzztest_data] { + foreach {interpreter scripts} [trd_fuzztest_data $buildname] { set bldDone 0 set subcmd [lrange $interpreter 1 end] set interpreter [lindex $interpreter 0] diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index 531eaaff7b..8a15080964 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -430,7 +430,7 @@ proc trd_extras {platform bld} { # Usage: # -# trd_fuzztest_data +# trd_fuzztest_data $buildname # # This returns data used by testrunner.tcl to run commands equivalent # to [make fuzztest]. The returned value is a list, which should be @@ -450,19 +450,25 @@ proc trd_extras {platform bld} { # directory containing this file). "fuzzcheck" and "sessionfuzz" have .exe # extensions on windows. # -proc trd_fuzztest_data {} { +proc trd_fuzztest_data {buildname} { set EXE "" set lFuzzDb [glob [file join $::testdir fuzzdata*.db]] set lSessionDb [glob [file join $::testdir sessionfuzz-data*.db]] + set sanBuilds {All-Debug Apple Have-Not Update-Delete-Limit} if {$::tcl_platform(platform) eq "windows"} { + if {$buildname eq "Windows-Win32Heap"} return [list] return [list fuzzcheck.exe $lFuzzDb] + } elseif {[lsearch $sanBuilds $buildname]>=0} { + return [list [trd_get_bin_name fuzzcheck] $lFuzzDb \ + [trd_get_bin_name fuzzcheck-asan] $lFuzzDb \ + [trd_get_bin_name fuzzcheck-ubsan] $lFuzzDb \ + {sessionfuzz run} $lSessionDb] + } else { + return [list [trd_get_bin_name fuzzcheck] $lFuzzDb \ + {sessionfuzz run} $lSessionDb] } - return [list [trd_get_bin_name fuzzcheck] $lFuzzDb \ - [trd_get_bin_name fuzzcheck-asan] $lFuzzDb \ - [trd_get_bin_name fuzzcheck-ubsan] $lFuzzDb \ - {sessionfuzz run} $lSessionDb] } From ece7709f189aaa3cff01f2ee8ba944ad87893990 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 29 Apr 2025 10:29:35 +0000 Subject: [PATCH 20/42] Do not run FUZZDB on either Windows-Memdebug or Windows-Win32Heap. FossilOrigin-Name: 10d8401d59136dda7051af0da398b06a18031560f2734f4fa347d339765d2089 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/testrunner.tcl | 2 ++ test/testrunner_data.tcl | 1 - 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 19f9e94aca..687299c173 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\stestrunner.tcl\smore\srestrictive\sabout\swhich\sconfigurations\srun\nfuzzcheck-asan\sand\sfuzzcheck-ubsan. -D 2025-04-29T01:37:31.258 +C Do\snot\srun\sFUZZDB\son\seither\sWindows-Memdebug\sor\sWindows-Win32Heap. +D 2025-04-29T10:29:35.813 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -1734,8 +1734,8 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl 3ee9722ea8024ad724cd3436ec7661a25bae716b3659a7f0f689c98dc1f40af0 x -F test/testrunner_data.tcl 81bee1515dde9f032c8e0169492a0428b6476dd7cb54276c237938e110db70d5 +F test/testrunner.tcl e84f0bbac16ec0d2f5bad2c7738ee7e9301ef8145f967d31a578407b2ca6219a x +F test/testrunner_data.tcl 8d5fa3851c48bc94e26db0be325202e44f6ca4ed838272b8d5b10c23817621e6 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3d8f3a255843c063157265702b76d40c38c774d29d658eea2d77efac92ac8789 -R f149be995be74f98448d3cbd3e7855b4 +P 785599394dde0fb36ae2e07d611f9523a05a0817dbd8c416582b3b682dbfa4ff +R d74a397dc8027d24c61598f2314a351f U drh -Z 469b94e6776e53378cf6fa455ca8fb8e +Z fe67492a73ff090eb49d8f6be601615a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0c8534eee2..317821a992 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -785599394dde0fb36ae2e07d611f9523a05a0817dbd8c416582b3b682dbfa4ff +10d8401d59136dda7051af0da398b06a18031560f2734f4fa347d339765d2089 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 68b7d99740..3a46264287 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1174,6 +1174,8 @@ proc add_fuzztest_jobs {buildname patternlist} { if {[string match fuzzcheck* $interpreter] && [info exists env(FUZZDB)] && [file readable $env(FUZZDB)] + && $buildname ne "Windows-Win32Heap" + && $buildname ne "Windows-Memdebug" } { set TRG(FUZZDB) $env(FUZZDB) set fname [file normalize $env(FUZZDB)] diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index 8a15080964..557c31b803 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -457,7 +457,6 @@ proc trd_fuzztest_data {buildname} { set sanBuilds {All-Debug Apple Have-Not Update-Delete-Limit} if {$::tcl_platform(platform) eq "windows"} { - if {$buildname eq "Windows-Win32Heap"} return [list] return [list fuzzcheck.exe $lFuzzDb] } elseif {[lsearch $sanBuilds $buildname]>=0} { return [list [trd_get_bin_name fuzzcheck] $lFuzzDb \ From c171d7624ee69bb29fc6615b40c636c8ead4d89f Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 29 Apr 2025 10:41:16 +0000 Subject: [PATCH 21/42] Improved --explain display of FUZZDB test plans in testrunner.tcl. FossilOrigin-Name: 42bdd7262ec4f8163263e0535c2cc35b526754d77e1f3ccf5b73b0f44d470d9c --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 687299c173..db0f69d9c3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\srun\sFUZZDB\son\seither\sWindows-Memdebug\sor\sWindows-Win32Heap. -D 2025-04-29T10:29:35.813 +C Improved\s--explain\sdisplay\sof\sFUZZDB\stest\splans\sin\stestrunner.tcl. +D 2025-04-29T10:41:16.849 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -1734,7 +1734,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl e84f0bbac16ec0d2f5bad2c7738ee7e9301ef8145f967d31a578407b2ca6219a x +F test/testrunner.tcl 038b538f04c857066b7057661878f4cc1517860f6c3f040087455780fc9f20dd x F test/testrunner_data.tcl 8d5fa3851c48bc94e26db0be325202e44f6ca4ed838272b8d5b10c23817621e6 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 785599394dde0fb36ae2e07d611f9523a05a0817dbd8c416582b3b682dbfa4ff -R d74a397dc8027d24c61598f2314a351f +P 10d8401d59136dda7051af0da398b06a18031560f2734f4fa347d339765d2089 +R f8d86c9e45d3a96c13f67cb70749dc31 U drh -Z fe67492a73ff090eb49d8f6be601615a +Z 1e9ce377b8e70ac62de3d9fa2977f8de # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 317821a992..9ef39976ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -10d8401d59136dda7051af0da398b06a18031560f2734f4fa347d339765d2089 +42bdd7262ec4f8163263e0535c2cc35b526754d77e1f3ccf5b73b0f44d470d9c diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 3a46264287..1248048103 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1701,7 +1701,13 @@ proc explain_layer {indent depid} { puts "${indent}$displayname in $dirname" explain_layer "${indent} " $jobid } elseif {$showtests} { - set tail [lindex $displayname end] + if {[lindex $displayname end-3] eq "--slice"} { + set M [lindex $displayname end-2] + set N [lindex $displayname end-1] + set tail "[lindex $displayname end] (slice $M/$N)" + } else { + set tail [lindex $displayname end] + } set e1 [lindex $displayname 1] if {[string match config=* $e1]} { set cfg [string range $e1 7 end] From 873fc5dff2a781251f2c9bd2c791a5fac45b7a2b Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 29 Apr 2025 11:35:51 +0000 Subject: [PATCH 22/42] Update testrunner.tcl documentation. Fix typo in testrunner.tcl output. FossilOrigin-Name: ba8800b4ba0a690c89d9fb96bb9a0d8789d04160ff9c47ea9afb2b403bec7f0b --- doc/testrunner.md | 71 +++++++++++++++++++++++++++++++++------------ manifest | 14 ++++----- manifest.uuid | 2 +- test/testrunner.tcl | 2 +- 4 files changed, 62 insertions(+), 27 deletions(-) diff --git a/doc/testrunner.md b/doc/testrunner.md index d0248573ee..c2808a1e48 100644 --- a/doc/testrunner.md +++ b/doc/testrunner.md @@ -15,6 +15,7 @@
  • 3.1. Commands to Run SQLite Tests
  • 3.2. Running ZipVFS Tests
  • 3.3. Investigating Source Code Test Failures +
  • 3.4. External Fuzzcheck Databases
  • 4. Extra testrunner.tcl Options
  • 5. Controlling CPU Core Utilization @@ -23,27 +24,30 @@ # 1. Overview -testrunner.tcl is a Tcl script used to run multiple SQLite tests using -multiple jobs. It supports the following types of tests: +The testrunner.tcl program is a Tcl script used to run multiple SQLite +tests in parallel, thus reducing testing time on multi-core machines. +It supports the following types of tests: * Tcl test scripts. + * Fuzzcheck tests, including using an external fuzzcheck database. + * Tests run with `make` commands. Examples: - - `make mdevtest` + - `make devtest` - `make releasetest` - `make sdevtest` - `make testrunner` -testrunner.tcl pipes the output of all tests and builds run into log file -**testrunner.log**, created in the current working directory. Search this -file to find details of errors. Suggested search commands: +The testrunner.tcl program stores output of all tests and builds run in +log file **testrunner.log**, created in the current working directory. +Search this file to find details of errors. Suggested search commands: * `grep "^!" testrunner.log` * `grep failed testrunner.log` -testrunner.tcl also populates SQLite database **testrunner.db**. This database -contains details of all tests run, running and to be run. A useful query -might be: +The testrunner.tcl program also populates SQLite database **testrunner.db**. +This database contains details of all tests run, running and to be run. +A useful query might be: ``` SELECT * FROM script WHERE state='failed' @@ -52,7 +56,7 @@ might be: Running the command: ``` - ./testfixture $(TESTDIR)/testrunner.tcl status + tclsh $(TESTDIR)/testrunner.tcl status ``` in the directory containing the testrunner.db database runs various queries @@ -60,7 +64,7 @@ to produce a succinct report on the state of a running testrunner.tcl script. Running: ``` - watch ./testfixture $(TESTDIR)/testrunner.tcl status + watch tclsh $(TESTDIR)/testrunner.tcl status ``` in another terminal is a good way to keep an eye on a long running test. @@ -239,9 +243,7 @@ of the specific tests run. As with source code tests, one or more patterns may be appended to any of the above commands (mdevtest, sdevtest or release). -In that case only Tcl tests (no fuzz or other tests) that match the specified -pattern are run. For example, to run the just the Tcl rtree tests in all -builds and configurations supported by "release": +Pattern matching is used for both Tcl tests and fuzz tests. ``` tclsh $TESTDIR/testrunner.tcl release rtree% @@ -292,6 +294,33 @@ target to build. This may be used either to run a `make` command test directly, or else to build a testfixture (or testfixture.exe) binary with which to run a Tcl test script, as described above. + +## 3.4 External Fuzzcheck Databases + +Testrunner.tcl will also run fuzzcheck against external (out of tree) +databases, for example fuzzcheck databases generated by dbsqlfuzz. To do +this, simply set the FUZZDB environment variable to the name of the external +database. For large external databases, testrunner.tcl will automatically use +the "`--slice`" command-line option of fuzzcheck to divide the work up between +multiple jobs, thus increasing parallelism. + +Thus, for example, to run a full releasetest including an external +dbsqlfuzz database, run a command like one of these: + +``` + FUZZDB=../fuzz/20250415.db test/testrunner.tcl releasetest + FUZZDB=../fuzz/20250415.db make releasetest + nmake /f Makefile.msc FUZZDB=../fuzz/20250415.db releasetest +``` + +The patternlist option to testrunner.tcl will match against fuzzcheck +databases. So if you want to run *only* tests involving the external +database, you can use a command something like this: + +``` + FUZZDB=../fuzz/20250415.db test/testrunner.tcl releasetest 20250415 +``` + # 4. Extra testrunner.tcl Options @@ -315,16 +344,22 @@ would normally execute into the testrunner.log file. Example: tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest" ``` -The **--explain** option is similar to --dryrun in that it prevents testrunner.tcl -from building any binaries or running any tests. The difference is that --explain -prints on standard output a human-readable summary of all the builds and tests that -would have been run. +The **--explain** option is similar to --dryrun in that it prevents +testrunner.tcl from building any binaries or running any tests. The +difference is that --explain prints on standard output a human-readable +summary of all the builds and tests that would have been run. ``` # Show what builds and tests would have been run tclsh $TESTDIR/testrunner.tcl --explain mdevtest ``` +The **--status** option uses VT100 escape sequences to display the test +status full-screen. This is similar to running +"`watch test/testrunner status`" in a separate window, just more convenient. +Unfortunately, this option does not work correctly on Windows, due to the +sketchy implementation of VT100 escapes on the Windows console. + # 5. Controlling CPU Core Utilization diff --git a/manifest b/manifest index db0f69d9c3..8616d04e15 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\s--explain\sdisplay\sof\sFUZZDB\stest\splans\sin\stestrunner.tcl. -D 2025-04-29T10:41:16.849 +C Update\stestrunner.tcl\sdocumentation.\s\sFix\stypo\sin\stestrunner.tcl\soutput. +D 2025-04-29T11:35:51.925 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -64,7 +64,7 @@ F doc/jsonb.md ede3238186e3a90bb79d20b2a6a06d0f9429a38e069e9da0efbad0f2ed48b924 F doc/lemon.html 89ea833a6f71773ab1a9063fbb7fb9b32147bc0b1057b53ecab94a3b30c0aef5 F doc/pager-invariants.txt 83aa3a4724b2d7970cc3f3461f0295c46d4fc19a835a5781cbb35cb52feb0577 F doc/tcl-extension-testing.md b88861804fc1eaf83249f8e206334189b61e150c360e1b80d0dcf91af82354f5 -F doc/testrunner.md 15583cf8c7d8a1c3378fd5d4319ca769a14c4d950a5df9b015d01d5be290dc69 +F doc/testrunner.md 41f3ab025a4e6f7505f476c9b3c01a65d109015dc4f4b23797d9c18bed4749be F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56 F doc/vfs-shm.txt 1a55f3f0e7b6745931b117ba5c9df3640d7a0536f532ef0052563100f4416f86 @@ -1734,7 +1734,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl 038b538f04c857066b7057661878f4cc1517860f6c3f040087455780fc9f20dd x +F test/testrunner.tcl dc0230b45cae84607dfcd5b8713966b47c0820fa4718a34d705e1423dedc0b0f x F test/testrunner_data.tcl 8d5fa3851c48bc94e26db0be325202e44f6ca4ed838272b8d5b10c23817621e6 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 10d8401d59136dda7051af0da398b06a18031560f2734f4fa347d339765d2089 -R f8d86c9e45d3a96c13f67cb70749dc31 +P 42bdd7262ec4f8163263e0535c2cc35b526754d77e1f3ccf5b73b0f44d470d9c +R 79541406257d79ca4afc3c688f589b5b U drh -Z 1e9ce377b8e70ac62de3d9fa2977f8de +Z 6a36b9638ff8ab1c4c150879edd48798 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9ef39976ab..1f12a6917f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42bdd7262ec4f8163263e0535c2cc35b526754d77e1f3ccf5b73b0f44d470d9c +ba8800b4ba0a690c89d9fb96bb9a0d8789d04160ff9c47ea9afb2b403bec7f0b diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 1248048103..a2896fd08f 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1647,7 +1647,7 @@ proc run_testset {} { puts "\nTest database is $TRG(dbname)" puts "Test log is $TRG(logname)" if {[info exists TRG(FUZZDB)]} { - puts "Extra fuzztest data taken from $TRG(FUZZDB)" + puts "Extra fuzzcheck data taken from $TRG(FUZZDB)" } trdb eval { SELECT sum(ntest) AS totaltest, From 2ba946b3417d6ca6130fc885849c4c8831ab852b Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 29 Apr 2025 14:23:21 +0000 Subject: [PATCH 23/42] Add the "--fuzzdb FILENAME" to testrunner.tcl as an alternative to setting the FUZZDB environment variable (as that can be awkward to do on Windows). Further improvements to the testrunner.tcl documentation. FossilOrigin-Name: 6fb84156a262ff89d1a2d1df6fbfac4c1a43fb55b9d15205508662e2c9b0894f --- doc/testrunner.md | 31 ++++++++++++++++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- test/testrunner.tcl | 4 ++++ 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/doc/testrunner.md b/doc/testrunner.md index c2808a1e48..d1696e9d1d 100644 --- a/doc/testrunner.md +++ b/doc/testrunner.md @@ -53,6 +53,14 @@ A useful query might be: SELECT * FROM script WHERE state='failed' ``` +You can get a summary of errors in a prior run by invoking commands like +these: + +``` + tclsh $(TESTDIR)/testrunner.tcl errors + tclsh $(TESTDIR)/testrunner.tcl errors -v +``` + Running the command: ``` @@ -61,13 +69,17 @@ Running the command: in the directory containing the testrunner.db database runs various queries to produce a succinct report on the state of a running testrunner.tcl script. -Running: +A good way to keep and eye on test progress is to run either of the two +following commands: ``` watch tclsh $(TESTDIR)/testrunner.tcl status + tclsh $(TESTDIR)/testrunner.tcl status -d 2 ``` -in another terminal is a good way to keep an eye on a long running test. +Both of the commands above accomplish about the same thing, but the second +one has the advantage of not requiring "watch" to be installed on your +system. Sometimes testrunner.tcl uses the `testfixture` binary that it is run with to run tests (see "Binary Tests" below). Sometimes it builds testfixture and @@ -297,18 +309,19 @@ run a Tcl test script, as described above. ## 3.4 External Fuzzcheck Databases -Testrunner.tcl will also run fuzzcheck against external (out of tree) -databases, for example fuzzcheck databases generated by dbsqlfuzz. To do -this, simply set the FUZZDB environment variable to the name of the external +Testrunner.tcl will also run fuzzcheck against an external (out of tree) +database, for example fuzzcheck databases generated by dbsqlfuzz. To do +this, simply add the "`--fuzzdb` *FILENAME*" command-line option or set +the FUZZDB environment variable to the name of the external database. For large external databases, testrunner.tcl will automatically use -the "`--slice`" command-line option of fuzzcheck to divide the work up between -multiple jobs, thus increasing parallelism. +the "`--slice`" command-line option of fuzzcheck to divide the work up into +multiple jobs, to increase parallelism. Thus, for example, to run a full releasetest including an external dbsqlfuzz database, run a command like one of these: ``` - FUZZDB=../fuzz/20250415.db test/testrunner.tcl releasetest + tclsh test/testrunner.tcl releasetest --fuzzdb ../fuzz/20250415.db FUZZDB=../fuzz/20250415.db make releasetest nmake /f Makefile.msc FUZZDB=../fuzz/20250415.db releasetest ``` @@ -318,7 +331,7 @@ databases. So if you want to run *only* tests involving the external database, you can use a command something like this: ``` - FUZZDB=../fuzz/20250415.db test/testrunner.tcl releasetest 20250415 + tclsh test/testrunner.tcl releasetest 20250415 --fuzzdb ../fuzz/20250415.db ``` diff --git a/manifest b/manifest index 8616d04e15..8613c0d0f7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stestrunner.tcl\sdocumentation.\s\sFix\stypo\sin\stestrunner.tcl\soutput. -D 2025-04-29T11:35:51.925 +C Add\sthe\s"--fuzzdb\sFILENAME"\sto\stestrunner.tcl\sas\san\salternative\sto\ssetting\nthe\sFUZZDB\senvironment\svariable\s(as\sthat\scan\sbe\sawkward\sto\sdo\son\sWindows).\nFurther\simprovements\sto\sthe\stestrunner.tcl\sdocumentation. +D 2025-04-29T14:23:21.489 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -64,7 +64,7 @@ F doc/jsonb.md ede3238186e3a90bb79d20b2a6a06d0f9429a38e069e9da0efbad0f2ed48b924 F doc/lemon.html 89ea833a6f71773ab1a9063fbb7fb9b32147bc0b1057b53ecab94a3b30c0aef5 F doc/pager-invariants.txt 83aa3a4724b2d7970cc3f3461f0295c46d4fc19a835a5781cbb35cb52feb0577 F doc/tcl-extension-testing.md b88861804fc1eaf83249f8e206334189b61e150c360e1b80d0dcf91af82354f5 -F doc/testrunner.md 41f3ab025a4e6f7505f476c9b3c01a65d109015dc4f4b23797d9c18bed4749be +F doc/testrunner.md 5ee928637e03f136a25fef852c5ed975932e31927bd9b05a574424ae18c31019 F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56 F doc/vfs-shm.txt 1a55f3f0e7b6745931b117ba5c9df3640d7a0536f532ef0052563100f4416f86 @@ -1734,7 +1734,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a -F test/testrunner.tcl dc0230b45cae84607dfcd5b8713966b47c0820fa4718a34d705e1423dedc0b0f x +F test/testrunner.tcl 614c4a28f7f730acd7bec53e17d76602fb480e0d538b6ec548169e03a093f92d x F test/testrunner_data.tcl 8d5fa3851c48bc94e26db0be325202e44f6ca4ed838272b8d5b10c23817621e6 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42bdd7262ec4f8163263e0535c2cc35b526754d77e1f3ccf5b73b0f44d470d9c -R 79541406257d79ca4afc3c688f589b5b +P ba8800b4ba0a690c89d9fb96bb9a0d8789d04160ff9c47ea9afb2b403bec7f0b +R 3acff940d07267222f56625ece4add87 U drh -Z 6a36b9638ff8ab1c4c150879edd48798 +Z f1457f63393e45549bde11151130d5cc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1f12a6917f..62a2f7c4d1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba8800b4ba0a690c89d9fb96bb9a0d8789d04160ff9c47ea9afb2b403bec7f0b +6fb84156a262ff89d1a2d1df6fbfac4c1a43fb55b9d15205508662e2c9b0894f diff --git a/test/testrunner.tcl b/test/testrunner.tcl index a2896fd08f..0c6982f426 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -98,6 +98,7 @@ Usage: --config CONFIGS Only use configs on comma-separate list CONFIGS --dryrun Write what would have happened to testrunner.log --explain Write summary to stdout + --fuzzdb FILENAME Additional external fuzzcheck database --jobs NUM Run tests using NUM separate processes --omit CONFIGS Omit configs on comma-separated list CONFIGS --status Show the full "status" report while running @@ -811,6 +812,9 @@ for {set ii 0} {$ii < [llength $argv]} {incr ii} { } elseif {($n>2 && [string match "$a*" --omit]) || $a=="-c"} { incr ii set TRG(omitconfig) [lindex $argv $ii] + } elseif {($n>2 && [string match "$a*" --fuzzdb])} { + incr ii + set env(FUZZDB) [lindex $argv $ii] } elseif {[string match "$a*" --stop-on-error]} { set TRG(stopOnError) 1 } elseif {[string match "$a*" --stop-on-coredump]} { From 5a9a2d85265efb88db00f998c11cfd456aa90baf Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 29 Apr 2025 16:30:58 +0000 Subject: [PATCH 24/42] Remove the run-fuzzcheck makefile targets, since testrunner now accomplishes that for us, and does a better job of it. FossilOrigin-Name: 3ffd867ed31cd0779aa92b5e8b71592a8e7224977f9da8e5d6793a78bbb80070 --- Makefile.msc | 4 -- main.mk | 113 -------------------------------------------------- manifest | 14 +++---- manifest.uuid | 2 +- 4 files changed, 8 insertions(+), 125 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 13207c0751..4428c4a65a 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1972,10 +1972,6 @@ fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) fuzzcheck-asan.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) /fsanitize=address $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -run-fuzzcheck: fuzzcheck.exe fuzzcheck-asan.exe - fuzzcheck --spinner $(FUZZDB) - fuzzcheck-asan --spinner $(FUZZDB) - ossshell.exe: $(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) diff --git a/main.mk b/main.mk index e553ffaaaf..53c0abb704 100644 --- a/main.mk +++ b/main.mk @@ -2244,119 +2244,6 @@ fuzzcheck-ubsan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) fuzzy: fuzzcheck-ubsan$(T.exe) xbin: fuzzcheck-ubsan$(T.exe) -# Usage: FUZZDB=filename make run-fuzzcheck -# -# Where filename is a fuzzcheck database, this target builds and runs -# fuzzcheck, fuzzcheck-asan, and fuzzcheck-ubsan on that database. -# -# FUZZDB can be a glob pattern of two or more databases. Example: -# -# FUZZDB=test/fuzzdata*.db make run-fuzzcheck -# -# The original rules for this target were like this: -# -# run-fuzzcheck: fuzzcheck$(T.exe) fuzzcheck-asan$(T.exe) fuzzcheck-ubsan$(T.exe) -# @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi -# ./fuzzcheck$(T.exe) --spinner $(FUZZDB) -# ./fuzzcheck-asan$(T.exe) --spinner $(FUZZDB) -# ./fuzzcheck-ubsan$(T.exe) --spinner $(FUZZDB) -# -# What follows is a decomposition of these rules in a way that allows make -# to run things in parallel when using the -jN option. -# -FUZZDB-check: - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi -run-fuzzcheck: run-fuzzcheck-n0 -run-fuzzcheck-n0: FUZZDB-check fuzzcheck$(T.exe) - ./fuzzcheck$(T.exe) --slice 0 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-n1 -run-fuzzcheck-n1: FUZZDB-check fuzzcheck$(T.exe) - ./fuzzcheck$(T.exe) --slice 1 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-n2 -run-fuzzcheck-n2: FUZZDB-check fuzzcheck$(T.exe) - ./fuzzcheck$(T.exe) --slice 2 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-n3 -run-fuzzcheck-n3: FUZZDB-check fuzzcheck$(T.exe) - ./fuzzcheck$(T.exe) --slice 3 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-n4 -run-fuzzcheck-n4: FUZZDB-check fuzzcheck$(T.exe) - ./fuzzcheck$(T.exe) --slice 4 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-n5 -run-fuzzcheck-n5: FUZZDB-check fuzzcheck$(T.exe) - ./fuzzcheck$(T.exe) --slice 5 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-n6 -run-fuzzcheck-n6: FUZZDB-check fuzzcheck$(T.exe) - ./fuzzcheck$(T.exe) --slice 6 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-n7 -run-fuzzcheck-n7: FUZZDB-check fuzzcheck$(T.exe) - ./fuzzcheck$(T.exe) --slice 7 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-n8 -run-fuzzcheck-n8: FUZZDB-check fuzzcheck$(T.exe) - ./fuzzcheck$(T.exe) --slice 8 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-n9 -run-fuzzcheck-n9: FUZZDB-check fuzzcheck$(T.exe) - ./fuzzcheck$(T.exe) --slice 9 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-a0 -run-fuzzcheck-a0: FUZZDB-check fuzzcheck-asan$(T.exe) - ./fuzzcheck-asan$(T.exe) --slice 0 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-a1 -run-fuzzcheck-a1: FUZZDB-check fuzzcheck-asan$(T.exe) - ./fuzzcheck-asan$(T.exe) --slice 1 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-a2 -run-fuzzcheck-a2: FUZZDB-check fuzzcheck-asan$(T.exe) - ./fuzzcheck-asan$(T.exe) --slice 2 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-a3 -run-fuzzcheck-a3: FUZZDB-check fuzzcheck-asan$(T.exe) - ./fuzzcheck-asan$(T.exe) --slice 3 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-a4 -run-fuzzcheck-a4: FUZZDB-check fuzzcheck-asan$(T.exe) - ./fuzzcheck-asan$(T.exe) --slice 4 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-a5 -run-fuzzcheck-a5: FUZZDB-check fuzzcheck-asan$(T.exe) - ./fuzzcheck-asan$(T.exe) --slice 5 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-a6 -run-fuzzcheck-a6: FUZZDB-check fuzzcheck-asan$(T.exe) - ./fuzzcheck-asan$(T.exe) --slice 6 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-a7 -run-fuzzcheck-a7: FUZZDB-check fuzzcheck-asan$(T.exe) - ./fuzzcheck-asan$(T.exe) --slice 7 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-a8 -run-fuzzcheck-a8: FUZZDB-check fuzzcheck-asan$(T.exe) - ./fuzzcheck-asan$(T.exe) --slice 8 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-a9 -run-fuzzcheck-a9: FUZZDB-check fuzzcheck-asan$(T.exe) - ./fuzzcheck-asan$(T.exe) --slice 9 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-u0 -run-fuzzcheck-u0: FUZZDB-check fuzzcheck-ubsan$(T.exe) - ./fuzzcheck-ubsan$(T.exe) --slice 0 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-u1 -run-fuzzcheck-u1: FUZZDB-check fuzzcheck-ubsan$(T.exe) - ./fuzzcheck-ubsan$(T.exe) --slice 1 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-u2 -run-fuzzcheck-u2: FUZZDB-check fuzzcheck-ubsan$(T.exe) - ./fuzzcheck-ubsan$(T.exe) --slice 2 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-u3 -run-fuzzcheck-u3: FUZZDB-check fuzzcheck-ubsan$(T.exe) - ./fuzzcheck-ubsan$(T.exe) --slice 3 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-u4 -run-fuzzcheck-u4: FUZZDB-check fuzzcheck-ubsan$(T.exe) - ./fuzzcheck-ubsan$(T.exe) --slice 4 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-u5 -run-fuzzcheck-u5: FUZZDB-check fuzzcheck-ubsan$(T.exe) - ./fuzzcheck-ubsan$(T.exe) --slice 5 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-u6 -run-fuzzcheck-u6: FUZZDB-check fuzzcheck-ubsan$(T.exe) - ./fuzzcheck-ubsan$(T.exe) --slice 6 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-u7 -run-fuzzcheck-u7: FUZZDB-check fuzzcheck-ubsan$(T.exe) - ./fuzzcheck-ubsan$(T.exe) --slice 7 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-u8 -run-fuzzcheck-u8: FUZZDB-check fuzzcheck-ubsan$(T.exe) - ./fuzzcheck-ubsan$(T.exe) --slice 8 10 $(FUZZDB) -run-fuzzcheck: run-fuzzcheck-u9 -run-fuzzcheck-u9: FUZZDB-check fuzzcheck-ubsan$(T.exe) - ./fuzzcheck-ubsan$(T.exe) --slice 9 10 $(FUZZDB) - ossshell$(T.exe): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h $(T.link) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ diff --git a/manifest b/manifest index 8613c0d0f7..8ca0a341ac 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Add\sthe\s"--fuzzdb\sFILENAME"\sto\stestrunner.tcl\sas\san\salternative\sto\ssetting\nthe\sFUZZDB\senvironment\svariable\s(as\sthat\scan\sbe\sawkward\sto\sdo\son\sWindows).\nFurther\simprovements\sto\sthe\stestrunner.tcl\sdocumentation. -D 2025-04-29T14:23:21.489 +C Remove\sthe\srun-fuzzcheck\smakefile\stargets,\ssince\stestrunner\snow\naccomplishes\sthat\sfor\sus,\sand\sdoes\sa\sbetter\sjob\sof\sit. +D 2025-04-29T16:30:58.194 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d F Makefile.in c3e414df4dc8dfb12f1f6baf129fcb6d18cd0ebd3c9109370fb3fceeeef9a37a F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 -F Makefile.msc bb2cc6f75bbcb2d690fbdd1489914a2febd5e99bad9c77538cb3330d304694c6 +F Makefile.msc a0941cadc1f6d953204864f68860c75e149e424e246343fb9753d4261dfaf547 F README.md e28077cfbef795e99c9c75ed95aa7257a1166709b562076441a8506ac421b7c1 F VERSION 001dea55eb8304ec9130b6b44a32d3fc349f279d45a7e224fc0730c3cb8e2372 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 @@ -709,7 +709,7 @@ F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk f03c2af6e15b4a24ae18e2f675da306cc8e95e3d064a9763bf50eff2e7d83b5f +F main.mk a41fa9e96ef139452d21a9ca5db4561ce053363e04d07da803f5e1cefb44c65e F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ba8800b4ba0a690c89d9fb96bb9a0d8789d04160ff9c47ea9afb2b403bec7f0b -R 3acff940d07267222f56625ece4add87 +P 6fb84156a262ff89d1a2d1df6fbfac4c1a43fb55b9d15205508662e2c9b0894f +R a1c49ae42fb2300a145738dcc51e2b94 U drh -Z f1457f63393e45549bde11151130d5cc +Z 2ea0a9b0d47db806738daf27dc577757 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 62a2f7c4d1..e718d721c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6fb84156a262ff89d1a2d1df6fbfac4c1a43fb55b9d15205508662e2c9b0894f +3ffd867ed31cd0779aa92b5e8b71592a8e7224977f9da8e5d6793a78bbb80070 From a20c09c477da10497459093cd6fc92f097378d69 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 29 Apr 2025 17:30:51 +0000 Subject: [PATCH 25/42] Upstream teaish for a tcl portability fix on Haiku and a much nicer impl of proj-tclConfig-sh-to-autosetup. FossilOrigin-Name: ca0d30a43b3dfb95dd6b491f592031a053b0b5e95361ffe01ec8bd56d5e2d110 --- autoconf/tea/Makefile.in | 5 +- autoconf/tea/autosetup/core.tcl | 79 +++++++++++++++----- autoconf/tea/autosetup/feature-tests.tcl | 2 +- autoconf/tea/teaish.tester.tcl.in | 4 +- autosetup/proj.tcl | 92 ++++++++++-------------- autosetup/sqlite-config.tcl | 2 +- manifest | 24 +++---- manifest.uuid | 2 +- 8 files changed, 122 insertions(+), 88 deletions(-) diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index 5aa374cf78..ad71c8b3e2 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -324,8 +324,9 @@ install-core: install-pre install-test: install-core @echo "Post-install test of [package require $(tx.name.pkg) $(tx.version)]..."; \ if echo \ - 'set c 0;' \ - 'if {[catch {package require $(tx.name.pkg) $(tx.version)}]} {incr c};' \ + 'set c 0; ' \ + '@TEAISH_POSTINST_PREREQUIRE@' \ + 'if {[catch {package require $(tx.name.pkg) $(tx.version)}]} {incr c};' \ 'exit $$c' \ | $(TCLSH) ; then \ echo "passed"; \ diff --git a/autoconf/tea/autosetup/core.tcl b/autoconf/tea/autosetup/core.tcl index c0baeb8fe5..4b3eb9a824 100644 --- a/autoconf/tea/autosetup/core.tcl +++ b/autoconf/tea/autosetup/core.tcl @@ -279,7 +279,7 @@ proc teaish-configure-core {} { # Set up some default values if the extension did not set them. # This must happen _after_ it's sourced but before # teaish-configure is called. - teaish__f2d_array f2d + array set f2d $::teaish__Config(pkginfo-f2d) foreach {pflag key type val} { - TEAISH_CFLAGS -v "" - TEAISH_LDFLAGS -v "" @@ -397,7 +397,7 @@ proc teaish-configure-core {} { # proc teaish-debug {msg} { if {$::teaish__Config(debug-enabled)} { - puts stderr [proj-bold "** DEBUG: \[[proj-current-scope 1]\]: $msg"] + puts stderr [proj-bold "** DEBUG: \[[proj-scope 1]\]: $msg"] } } @@ -831,8 +831,60 @@ proc teaish__check_tcl {} { } } msg-result "Using Tcl [get-define TCL_VERSION] from [get-define TCL_PREFIX]." + teaish__tcl_platform_quirks }; # teaish__check_tcl +# +# Perform last-minute platform-specific tweaks to account for quirks. +# +proc teaish__tcl_platform_quirks {} { + define TEAISH_POSTINST_PREREQUIRE "" + switch -glob -- [get-define host] { + *-haiku { + # Haiku's default TCLLIBDIR is "all wrong": it points to a + # read-only virtual filesystem mount-point. We bend it back to + # fit under $TCL_PACKAGE_PATH here. + foreach {k d} { + vj TCL_MAJOR_VERSION + vn TCL_MINOR_VERSION + pp TCL_PACKAGE_PATH + ld TCLLIBDIR + } { + set $k [get-define $d] + } + if {[string match /packages/* $ld]} { + set old $ld + set tail [file tail $ld] + if {8 == $vj} { + set ld "${pp}/tcl${vj}.${vn}/${tail}" + } else { + proj-assert {9 == $vj} + set ld "${pp}/${tail}" + } + define TCLLIBDIR $ld + # [load foo.so], without a directory part, does not work via + # automated tests on Haiku (but works when run + # manually). Similarly, the post-install [package require ...] + # test fails, presumably for a similar reason. We work around + # the former in teaish.tester.tcl.in. We work around the + # latter by amending the post-install check's ::auto_path (in + # Makefile.in). This code MUST NOT contain any single-quotes. + define TEAISH_POSTINST_PREREQUIRE \ + [join [list set ::auto_path \ + \[ linsert \$::auto_path 0 $ld \] \; \ + ]] + proj-indented-notice [subst -nocommands -nobackslashes { + Haiku users take note: patching target installation dir to match + Tcl's home because Haiku's is not writable. + + Original : $old + Substitute: $ld + }] + } + } + } +}; # teaish__tcl_platform_quirks + # # Searches $::argv and/or the build dir and/or the source dir for # teaish.tcl and friends. Fails if it cannot find teaish.tcl or if @@ -1246,7 +1298,7 @@ proc teaish-dist-add {args} { #} # -# @teash-append-make args... +# @teash-make-add args... # # Appends makefile code to the TEAISH_MAKEFILE_CODE define. Each # arg may be any of: @@ -1331,7 +1383,7 @@ proc teaish-make-obj {o src args} { } } teaish-make-add \ - "# [proj-current-scope 1] -> [proj-current-scope] $o $src" -nl \ + "# [proj-scope 1] -> [proj-scope] $o $src" -nl \ "$o: $src $::teaish__Config(teaish.tcl)" if {[info exists flags(-deps)]} { teaish-make-add " " [join $flags(-deps)] @@ -1378,7 +1430,7 @@ proc teaish-make-clean {args} { } } set rule [teaish__cleanup_rule $tgt] - teaish-make-add "# [proj-current-scope 1] -> [proj-current-scope]: [join $args]\n" + teaish-make-add "# [proj-scope 1] -> [proj-scope]: [join $args]\n" teaish-make-add "${rule}:\n\trm ${rmflags}" foreach a $args { teaish-make-add " \"$a\"" @@ -1461,7 +1513,7 @@ proc teaish-check-cached {args} { } lassign $args msg script if {"" eq $msg} { - set msg [proj-current-scope 1] + set msg [proj-scope 1] } msg-checking "${msg} ... " if {[teaish-feature-cache-check 1 check]} { @@ -1583,7 +1635,7 @@ proc teaish__defs_format {type value} { default { proj-error \ "Unknown [project-current-scope] -type ($type) called from" \ - [proj-current-scope 1] + [proj-scope 1] } } return $value @@ -1850,7 +1902,7 @@ proc teaish-pkginfo-set {args} { } proj-parse-simple-flags args flags $flagDefs if {[llength $args]} { - proj-error -up "Too many (or unknown) arguments to [proj-current-scope]: $args" + proj-error -up "Too many (or unknown) arguments to [proj-scope]: $args" } foreach {f d} $::teaish__Config(pkginfo-f2d) { if {$sentinel eq [set v $flags($f)]} continue @@ -1964,13 +2016,12 @@ proc teaish-pkginfo-get {args} { } default { - proj-error "invalid arg count from [proj-current-scope 1]" + proj-error "invalid arg count from [proj-scope 1]" } } - set cases [join $cases] foreach {flag defName} $::teaish__Config(pkginfo-f2d) { - switch -exact -- $flag $cases + switch -exact -- $flag [join $cases] } if {0 == $argc} { return $rv } } @@ -2080,12 +2131,6 @@ proc teaish-get {flag} { proj-error "Unhandled flag: $flag" } -array set ::teaish__PkgInfoF2D $::teaish__Config(pkginfo-f2d) -proc teaish__f2d_array {tgtArrayName} { - upvar $tgtArrayName tgt - set tgt $::teaish__PkgInfoF2D -} - # # Handles --teaish-create-extension=TARGET-DIR # diff --git a/autoconf/tea/autosetup/feature-tests.tcl b/autoconf/tea/autosetup/feature-tests.tcl index 9d77092c82..6c927d1a77 100644 --- a/autoconf/tea/autosetup/feature-tests.tcl +++ b/autoconf/tea/autosetup/feature-tests.tcl @@ -186,7 +186,7 @@ proc teaish-import-features {args} { # if {$flags(-run) && "" ne $pk} { proj-error "Cannot use both -run and $pk" \ - " (called from [proj-current-scope 1])" + " (called from [proj-scope 1])" } foreach arg $args { diff --git a/autoconf/tea/teaish.tester.tcl.in b/autoconf/tea/teaish.tester.tcl.in index 802f1d9754..4e203cd785 100644 --- a/autoconf/tea/teaish.tester.tcl.in +++ b/autoconf/tea/teaish.tester.tcl.in @@ -17,7 +17,9 @@ @TEAISH_VSATISFIES_CODE@ @endif if {[llength [lindex $::argv 0]] > 0} { - load [lindex $::argv 0] [lindex $::argv 1]; + load [file normalize [lindex $::argv 0]] [lindex $::argv 1]; + # ----^^^^^^^ needed on Haiku when argv 0 is just a filename, else + # load cannot find the file. } source -encoding utf-8 [lindex $::argv 2]; # teaish/tester.tcl @if TEAISH_PKGINIT_TCL diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 7690486cf2..4691cfe36f 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -11,7 +11,7 @@ # # ----- @module proj.tcl ----- -# @section Project Helper APIs +# @section Project-agnostic Helper APIs # # @@ -85,7 +85,7 @@ set proj__Config(isatty) [isatty? stdout] # proc proj-warn {args} { show-notices - puts stderr [join [list "WARNING: \[[proj-current-scope 1]\]: " {*}$args] " "] + puts stderr [join [list "WARNING: \[[proj-scope 1]\]: " {*}$args] " "] } # @@ -105,7 +105,7 @@ proc proj-fatal {args} { set args [lassign $args -] incr lvl } - puts stderr [join [list "FATAL: \[[proj-current-scope $lvl]]: " {*}$args]] + puts stderr [join [list "FATAL: \[[proj-scope $lvl]]: " {*}$args]] exit 1 } @@ -121,27 +121,26 @@ proc proj-error {args} { set args [lassign $args -] incr lvl } - error [join [list "\[[proj-current-scope $lvl]]:" {*}$args]] + error [join [list "\[[proj-scope $lvl]]:" {*}$args]] } # # @proj-assert script ?message? # -# Kind of like a C assert: if uplevel (eval) of [expr {$script}] is -# false, a fatal error is triggered. The error message, by default, -# includes the body of the failed assertion, but if $msg is set then -# that is used instead. +# Kind of like a C assert: if uplevel of [list expr $script] is false, +# a fatal error is triggered. The error message, by default, includes +# the body of the failed assertion, but if $msg is set then that is +# used instead. # proc proj-assert {script {msg ""}} { if {1 == [get-env proj-assert 0]} { msg-result [proj-bold "asserting: $script"] } - set x "expr \{ $script \}" - if {![uplevel 1 $x]} { + if {![uplevel 1 [list expr $script]]} { if {"" eq $msg} { set msg $script } - proj-fatal "Assertion failed in \[[proj-current-scope 1]\]: $msg" + proj-fatal "Assertion failed in \[[proj-scope 1]\]: $msg" } } @@ -214,7 +213,7 @@ proc proj-indented-notice {args} { # Returns 1 if cross-compiling, else 0. # proc proj-is-cross-compiling {} { - return [expr {[get-define host] ne [get-define build]}] + expr {[get-define host] ne [get-define build]} } # @@ -251,7 +250,7 @@ proc proj-cflags-without-werror {{var CFLAGS}} { default { lappend rv $f } } } - return [join $rv " "] + join $rv " " } # @@ -377,7 +376,8 @@ proc proj-first-bin-of {args} { set rc "" foreach b $args { set u [string toupper $b] - # Note that cc-path-progs defines $u to false if it finds no match. + # Note that cc-path-progs defines $u to "false" if it finds no + # match. if {[cc-path-progs $b]} { set rc [get-define $u] } @@ -521,7 +521,7 @@ proc proj-define-for-opt {flag def {msg ""} {iftrue 1} {iffalse 0}} { # @proj-opt-define-bool ?-v? optName defName ?descr? # # Checks [proj-opt-truthy $optName] and calls [define $defName X] -# where X is 0 for false and 1 for true. descr is an optional +# where X is 0 for false and 1 for true. $descr is an optional # [msg-checking] argument which defaults to $defName. Returns X. # # If args[0] is -v then the boolean semantics are inverted: if @@ -531,7 +531,7 @@ proc proj-define-for-opt {flag def {msg ""} {iftrue 1} {iffalse 0}} { proc proj-opt-define-bool {args} { set invert 0 if {[lindex $args 0] eq "-v"} { - set invert 1 + incr invert lassign $args - optName defName descr } else { lassign $args optName defName descr @@ -542,14 +542,9 @@ proc proj-opt-define-bool {args} { #puts "optName=$optName defName=$defName descr=$descr" set rc 0 msg-checking "[join $descr] ... " - if {[proj-opt-truthy $optName]} { - if {0 eq $invert} { - set rc 1 - } else { - set rc 0 - } - } elseif {0 ne $invert} { - set rc 1 + set rc [proj-opt-truthy $optName] + if {$invert} { + set rc [expr {!$rc}] } msg-result $rc define $defName $rc @@ -1343,7 +1338,7 @@ proc proj-dump-defs-json {file args} { # that [opt-value canonical] will return X if --alias=X is passed to # configure. # -# That said: autosetup's [opt-src] does support alias forms, but it +# That said: autosetup's [opt-str] does support alias forms, but it # requires that the caller know all possible aliases. It's simpler, in # terms of options handling, if there's only a single canonical name # which each down-stream call of [opt-...] has to know. @@ -1503,11 +1498,11 @@ proc proj-env-file {flag {dflt ""}} { # If none of those are set, $dflt is returned. # proc proj-get-env {var {dflt ""}} { - return [get-env $var [proj-env-file $var $dflt]] + get-env $var [proj-env-file $var $dflt] } # -# @proj-current-scope ?lvl? +# @proj-scope ?lvl? # # Returns the name of the _calling_ proc from ($lvl + 1) levels up the # call stack (where the caller's level will be 1 up from _this_ @@ -1515,7 +1510,7 @@ proc proj-get-env {var {dflt ""}} { # returned and if it would be negative then a string indicating such # is returned (as opposed to throwing an error). # -proc proj-current-scope {{lvl 0}} { +proc proj-scope {{lvl 0}} { #uplevel [expr {$lvl + 1}] {lindex [info level 0] 0} set ilvl [info level] set offset [expr {$ilvl - $lvl - 1}] @@ -1528,6 +1523,15 @@ proc proj-current-scope {{lvl 0}} { } } +# +# Deprecated name of [proj-scope]. +# +proc proj-current-scope {{lvl 0}} { + puts stderr \ + "Deprecated proj-current-scope called from [proj-scope 1]. Use proj-scope instead." + proj-scope [incr lvl] +} + # # Converts parts of tclConfig.sh to autosetup [define]s. # @@ -1548,6 +1552,7 @@ proc proj-tclConfig-sh-to-autosetup {tclConfigSh} { TCL_VERSION TCL_MAJOR_VERSION TCL_MINOR_VERSION + TCL_PACKAGE_PATH TCL_PATCH_LEVEL TCL_SHLIB_SUFFIX } @@ -1564,26 +1569,7 @@ proc proj-tclConfig-sh-to-autosetup {tclConfigSh} { lappend shBody "exit" set shBody [join $shBody "\n"] #puts "shBody=$shBody\n"; exit - if {0} { - # This doesn't work but would be preferable to using a temp file... - set fd [open "| sh" "rw"] - #puts "fd = $fd"; exit - puts $fd $shBody - #flush $fd; # "bad file descriptor"? Without flush, [read] blocks - set rd [read $fd] - close $fd - puts "rd=$rd"; exit 1 - eval $rd - } else { - set shName ".tclConfigSh.tcl" - proj-file-write $shName $shBody - catch { - eval [exec sh $shName $tclConfigSh] - expr 1 - } rc xopts - file delete -force $shName - return {*}$xopts $rc - } + eval [exec echo $shBody | sh] } # @@ -1662,7 +1648,7 @@ proc proj-dot-ins-append {fileIn args} { proj-fatal "Too many arguments: $fileIn $args" } } - #puts "******* [proj-current-scope]: adding $fileIn" + #puts "******* [proj-scope]: adding $fileIn" lappend ::proj__Config(dot-in-files) $fileIn } @@ -1706,7 +1692,7 @@ proc proj-dot-ins-process {args} { -validate 0 {expr 1} } if {[llength $args] > 0} { - error "Invalid argument to [proj-current-scope]: $args" + error "Invalid argument to [proj-scope]: $args" } foreach f $::proj__Config(dot-in-files) { proj-assert {3==[llength $f]} \ @@ -1844,7 +1830,7 @@ proc proj-define-amend {args} { } } if {"" eq $defName} { - proj-error "Missing defineName argument in call from [proj-current-scope 1]" + proj-error "Missing defineName argument in call from [proj-scope 1]" } if {$isdefs} { set args $xargs @@ -1975,7 +1961,7 @@ array set proj__Cache {} # Returns a cache key for the given argument: # # integer: relative call stack levels to get the scope name of for -# use as a key. [proj-current-scope [expr {1 + $arg + addLevel}]] is +# use as a key. [proj-scope [expr {1 + $arg + addLevel}]] is # then used to generate the key. i.e. the default of 0 uses the # calling scope's name as the key. # @@ -1986,7 +1972,7 @@ array set proj__Cache {} proc proj-cache-key {{addLevel 0} arg} { if {"-" eq $arg} {set arg 0} if {[string is integer -strict $arg]} { - return [proj-current-scope [expr {$arg + $addLevel + 1}]] + return [proj-scope [expr {$arg + $addLevel + 1}]] } return $arg } diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index 74d5b79317..1df6e233ff 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -538,7 +538,7 @@ proc msg-debug {msg} { # the debug message. It is not legal to call this from the global # scope. proc proc-debug {msg} { - msg-debug "\[[proj-current-scope 1]\]: $msg" + msg-debug "\[[proj-scope 1]\]: $msg" } define OPT_FEATURE_FLAGS {} ; # -DSQLITE_OMIT/ENABLE flags. diff --git a/manifest b/manifest index 8ca0a341ac..aa62d6bae4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\srun-fuzzcheck\smakefile\stargets,\ssince\stestrunner\snow\naccomplishes\sthat\sfor\sus,\sand\sdoes\sa\sbetter\sjob\sof\sit. -D 2025-04-29T16:30:58.194 +C Upstream\steaish\sfor\sa\stcl\sportability\sfix\son\sHaiku\sand\sa\smuch\snicer\simpl\sof\sproj-tclConfig-sh-to-autosetup. +D 2025-04-29T17:30:51.828 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -22,12 +22,12 @@ F autoconf/Makefile.msc 5bc67d3912444c40c6f96d003e5c90663e51abb83d204a520110b1b2 F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac -F autoconf/tea/Makefile.in 6aebb3d820c8aab94f7599e8594d5f2ecea63894d08f36ba4141040b322042bc +F autoconf/tea/Makefile.in 8c00e2ed350754d6b45681318ed7e4578aed8ad732abcac0593c1b10dc29e5a6 F autoconf/tea/README.txt 656d4686c509d375f5988ff3deda94f65fe6cd8358cd55d1f1dcc7b6e2ff73aa F autoconf/tea/auto.def ce95b9450e2fa4ba5dc857e208fe10f4e6f2d737796ac3278aee6079db417529 F autoconf/tea/autosetup/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca -F autoconf/tea/autosetup/core.tcl f0eebd4aa416df44f0c22f7be1a2b5b05e8fe4f64e53b0d06a21f328963c5109 -F autoconf/tea/autosetup/feature-tests.tcl 307cac35806817fc87bd0d92004364ee682c0e99f2ab505291500493ab6c7a5f +F autoconf/tea/autosetup/core.tcl 7d942639871111e2fcef571c9d5a6e2dc75972eb214cf814a6b99f1e2b25182f +F autoconf/tea/autosetup/feature-tests.tcl 18194fb79a24d30e5bbdeab40999616f39278b53a27525349ded033af2fd73be F autoconf/tea/autosetup/tester.tcl c293695a0ab5d9e8d0ceeb0ee422f90e8a6aa9f0c7c51acd0b6d9f09d8edfed3 F autoconf/tea/configure d0b12b984edca6030d1976375b80157ac78b5b90a5b4f0dcee39357f63f4a80b x F autoconf/tea/doc/sqlite3.n 9a97f4f717ceab73004ea412af7960625c1cb24b5c25e4ae4c8b5d8fa4300f4e @@ -35,7 +35,7 @@ F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in e07da6b94561f4aa382bab65b1ccceb04701b97bf59d007c1d1f20a222b22d07 F autoconf/tea/teaish.tcl 8e124f33cbaf9309f3e49be4e7018a03b8f3a52f8c8d9e1e5419f4f7b0eae59e F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 -F autoconf/tea/teaish.tester.tcl.in 0d048e5569ad9bbaffbe5123c2084f9084d424a7719d06f5ca941caafbcca320 +F autoconf/tea/teaish.tester.tcl.in 31ac5b7b1e226b7e1bfc6b578a6c1a51550306ef7afae5949eec046df006ca7d F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e F autosetup/README.md f324bb9f9bf1cc787122034df53fbfdfed28ee2657e6652b763d992ab0d04829 @@ -51,8 +51,8 @@ F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e F autosetup/jimsh0.c 563b966c137a4ce3c9333e5196723b7ac0919140a9d7989eb440463cd855c367 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 4deaa5bba71390367a55fe4f0dbdc0e0ddbefb5c24a0ee4a5a54bbf288a3285d -F autosetup/sqlite-config.tcl 54c88abadda1a42c6276186afcf6534e9bcb08350d96676c4ec08b99f2ada484 +F autosetup/proj.tcl 0287234d817e800ab0e10d46bf98545ba5762edd69e5dd0e2902029a7e6c3555 +F autosetup/sqlite-config.tcl 7ff986f6c3951f3aec5608522cbf772d8d04a0d26cc894289e2ca4836e018719 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6fb84156a262ff89d1a2d1df6fbfac4c1a43fb55b9d15205508662e2c9b0894f -R a1c49ae42fb2300a145738dcc51e2b94 -U drh -Z 2ea0a9b0d47db806738daf27dc577757 +P 3ffd867ed31cd0779aa92b5e8b71592a8e7224977f9da8e5d6793a78bbb80070 +R 83f48c22e5e2f14e3dc7e22807086c00 +U stephan +Z fd9d0c6c436c24b0f01cb0cb367bf38e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e718d721c9..fcf54df0f3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ffd867ed31cd0779aa92b5e8b71592a8e7224977f9da8e5d6793a78bbb80070 +ca0d30a43b3dfb95dd6b491f592031a053b0b5e95361ffe01ec8bd56d5e2d110 From 88ed1806a4dae310ea08a9c2e6c5631e87c4fabb Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 30 Apr 2025 12:48:20 +0000 Subject: [PATCH 26/42] Fix an issue in Bloom filters on RHS subsqueries to IN operators. See [forum:/forumpost/792a09cb3df9e69f|forum post 792a09cb3d] for a description of the problem. Also improve comments related to [baa83b460c677c21] which was origin of the problem. FossilOrigin-Name: cdef486e212fe4b26605065d9cff08f608cb80df48ee64e4be63637769bdfacc --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 9 +++++---- src/vdbe.c | 8 +++++++- test/bloom1.test | 14 +++++++++++++- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index aa62d6bae4..01babf00ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Upstream\steaish\sfor\sa\stcl\sportability\sfix\son\sHaiku\sand\sa\smuch\snicer\simpl\sof\sproj-tclConfig-sh-to-autosetup. -D 2025-04-29T17:30:51.828 +C Fix\san\sissue\sin\sBloom\sfilters\son\sRHS\ssubsqueries\sto\sIN\soperators.\nSee\s[forum:/forumpost/792a09cb3df9e69f|forum\spost\s792a09cb3d]\sfor\na\sdescription\sof\sthe\sproblem.\s\sAlso\simprove\scomments\srelated\nto\s[baa83b460c677c21]\swhich\swas\sorigin\sof\sthe\sproblem. +D 2025-04-30T12:48:20.768 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -736,7 +736,7 @@ F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9 F src/dbpage.c fcb1aafe00872a8aff9a7aa0ef7ff1b01e5817ec7bbd521f8f3e1e674ac8d609 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c 61c3baab38f1b50eb4696e1f37c8f7ae1d1ecbfc1a35d446cfd1886624784131 +F src/expr.c 565d2b6403e85126d2f39f993bfbd9968c29b1100450357bc6da931469fad315 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f F src/func.c 7686ea382b20e8bfe2ab9de76150c99ee7b6e83523561f3c7787e0f68cb435c2 @@ -854,7 +854,7 @@ F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c 3a20cbae9688af4c1e3754cc2520189d00762e37f60c2deb0b303360d166bba6 F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 F src/vacuum.c d580ceb395c1ae3d59da41cbfea60683ff7dd2b94ddf4d0f5657620159e2eeb7 -F src/vdbe.c eafa2e2ab45edc36c378c5a6f8d7083d30692c96a1f03945808895414dc5d347 +F src/vdbe.c 6e8030369862a64e5de35cf62a53d466ee4e9d54f4cce1219eca8914b96e956c F src/vdbe.h 31eddcffc1d14c76c2a20fe4e137e1ee43d44f370896fae14a067052801a3625 F src/vdbeInt.h 5446f60e89b2aa7cdf3ab0ec4e7b01b8732cd9d52d9092a0b8b1bf700768f784 F src/vdbeapi.c 28fab30ed0acc981aecfdcaab0a421503609078e29850eb28494816682baf0a7 @@ -977,7 +977,7 @@ F test/bind2.test 918bc35135f4141809ead7585909cde57d44db90a7a62aef540127148f91aa F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 F test/blob.test e7ac6c7d3a985cc4678c64f325292529a69ae252 -F test/bloom1.test 04f3a17df8912bfdc292c41b59d79f93893fe69799f3089a64451f9112f9658f +F test/bloom1.test 3b6277a647ac503b5d5df331037b0c01c40e88cc9537b94eaf2d8aa334ed4c8f F test/boundary1.tcl 6421b2d920d8b09539503a8673339d32f7609eb1 F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3ffd867ed31cd0779aa92b5e8b71592a8e7224977f9da8e5d6793a78bbb80070 -R 83f48c22e5e2f14e3dc7e22807086c00 -U stephan -Z fd9d0c6c436c24b0f01cb0cb367bf38e +P ca0d30a43b3dfb95dd6b491f592031a053b0b5e95361ffe01ec8bd56d5e2d110 +R a016079ed04d648dbdfc37d62a0ccf45 +U drh +Z 4df416cb1305012d324f8e35a43340cd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fcf54df0f3..4a25d7d529 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca0d30a43b3dfb95dd6b491f592031a053b0b5e95361ffe01ec8bd56d5e2d110 +cdef486e212fe4b26605065d9cff08f608cb80df48ee64e4be63637769bdfacc diff --git a/src/expr.c b/src/expr.c index dd5ea20383..bf15811bc4 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3626,11 +3626,12 @@ void sqlite3CodeRhsOfIN( sqlite3SelectDelete(pParse->db, pCopy); sqlite3DbFree(pParse->db, dest.zAffSdst); if( addrBloom ){ + /* Remember that location of the Bloom filter in the P3 operand + ** of the OP_Once that began this subroutine. tag-202407032019 */ sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; if( dest.iSDParm2==0 ){ - sqlite3VdbeChangeToNoop(v, addrBloom); - }else{ - sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; + /* If the Bloom filter won't actually be used, keep it small */ + sqlite3VdbeGetOp(v, addrBloom)->p1 = 10; } } if( rc ){ @@ -4077,7 +4078,7 @@ static void sqlite3ExprCodeIN( if( ExprHasProperty(pExpr, EP_Subrtn) ){ const VdbeOp *pOp = sqlite3VdbeGetOp(v, pExpr->y.sub.iAddr); assert( pOp->opcode==OP_Once || pParse->nErr ); - if( pOp->opcode==OP_Once && pOp->p3>0 ){ + if( pOp->opcode==OP_Once && pOp->p3>0 ){ /* tag-202407032019 */ assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, rLhs, nVector); VdbeCoverage(v); diff --git a/src/vdbe.c b/src/vdbe.c index d4009b70f6..c9972fe16b 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2644,7 +2644,7 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ break; } -/* Opcode: Once P1 P2 * * * +/* Opcode: Once P1 P2 P3 * * ** ** Fall through to the next instruction the first time this opcode is ** encountered on each invocation of the byte-code program. Jump to P2 @@ -2660,6 +2660,12 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ ** whether or not the jump should be taken. The bitmask is necessary ** because the self-altering code trick does not work for recursive ** triggers. +** +** The P3 operand is not used directly by this opcode. However P3 is +** used by the code generator as follows: If this opcode is the start +** of a subroutine and that subroutine uses a Bloom filter, then P3 will +** be the register that holds that Bloom filter. See tag-202407032019 +** in the source code for implementation details. */ case OP_Once: { /* jump */ u32 iAddr; /* Address of this instruction */ diff --git a/test/bloom1.test b/test/bloom1.test index f8efcc1846..09553c3b9b 100644 --- a/test/bloom1.test +++ b/test/bloom1.test @@ -224,6 +224,18 @@ do_execsql_test 5.3 { SELECT 0 as c_0 ); } {0} - + +# 2025-04-30 https://sqlite.org/forum/forumpost/792a09cb3df9e69f +# A continuation of the above. +# +do_execsql_test 6.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a); + SELECT 111 IN ( + SELECT 222 FROM (SELECT 333 ORDER BY 1) + UNION ALL + SELECT 444 FROM (SELECT 555 FROM t1 ORDER BY 1) + ); +} 0 finish_test From f23a61258bf9f9e346fe33ad6f740834455ade56 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 30 Apr 2025 14:37:00 +0000 Subject: [PATCH 27/42] Fix a harmless problem in the CLI in which SQL errors that occur during the ".schema" command are properly ignored, yes still appear in the ".log" output. [forum:/forumpost/42fe6520b803be51|Forum post 42fe6520b8] FossilOrigin-Name: 20abf1ec107f942e4527901685d61283c9c2fe7bcefad63dbf5c6cbf050da849 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/shell.c.in | 59 +++++++++++++++++++++++++++++--------------------- 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index 01babf00ef..d5ac399b08 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\sin\sBloom\sfilters\son\sRHS\ssubsqueries\sto\sIN\soperators.\nSee\s[forum:/forumpost/792a09cb3df9e69f|forum\spost\s792a09cb3d]\sfor\na\sdescription\sof\sthe\sproblem.\s\sAlso\simprove\scomments\srelated\nto\s[baa83b460c677c21]\swhich\swas\sorigin\sof\sthe\sproblem. -D 2025-04-30T12:48:20.768 +C Fix\sa\sharmless\sproblem\sin\sthe\sCLI\sin\swhich\sSQL\serrors\sthat\soccur\sduring\nthe\s".schema"\scommand\sare\sproperly\signored,\syes\sstill\sappear\sin\sthe\s".log"\noutput.\s[forum:/forumpost/42fe6520b803be51|Forum\spost\s42fe6520b8] +D 2025-04-30T14:37:00.121 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -786,7 +786,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 20e1fbe8f840ffc0cd835e33f68a802a22e34faa918d7a269f3de242fda02f99 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 1a5956231f7c57571288eaad61e5c37aaf0f3acb5c8a5ea0b896938166b62fa2 -F src/shell.c.in 1e8b9bf369e80cdf9b029142e773038bc12bd38aea1c56df4af6bf7b46cae955 +F src/shell.c.in 2c904da4431fed365e7d5029b8cb2da46cb3e8cf8a09305d1478836a2301ea96 F src/sqlite.h.in 22882ddd3a70751aa8864c81993ee4562ed54c2c508b6270f75e223ffee38e1b F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ca0d30a43b3dfb95dd6b491f592031a053b0b5e95361ffe01ec8bd56d5e2d110 -R a016079ed04d648dbdfc37d62a0ccf45 +P cdef486e212fe4b26605065d9cff08f608cb80df48ee64e4be63637769bdfacc +R ff4dbf6198636e450976250aaa2d8234 U drh -Z 4df416cb1305012d324f8e35a43340cd +Z 88680290faafcb12b4513cbfd1fe3d4f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4a25d7d529..3d3fb0bd6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cdef486e212fe4b26605065d9cff08f608cb80df48ee64e4be63637769bdfacc +20abf1ec107f942e4527901685d61283c9c2fe7bcefad63dbf5c6cbf050da849 diff --git a/src/shell.c.in b/src/shell.c.in index ca76e4a6e9..7af8f79b8d 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1243,30 +1243,6 @@ static void shellDtostr( sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); } - -/* -** SQL function: shell_module_schema(X) -** -** Return a fake schema for the table-valued function or eponymous virtual -** table X. -*/ -static void shellModuleSchema( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - const char *zName; - char *zFake; - UNUSED_PARAMETER(nVal); - zName = (const char*)sqlite3_value_text(apVal[0]); - zFake = zName? shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName) : 0; - if( zFake ){ - sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), - -1, sqlite3_free); - free(zFake); - } -} - /* ** SQL function: shell_add_schema(S,X) ** @@ -5710,6 +5686,39 @@ static void shellUSleepFunc( sqlite3_result_int(context, sleep); } +/* +** SQL function: shell_module_schema(X) +** +** Return a fake schema for the table-valued function or eponymous virtual +** table X. +*/ +static void shellModuleSchema( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zName; + char *zFake; + ShellState *p = (ShellState*)sqlite3_user_data(pCtx); + FILE *pSavedLog = p->pLog; + UNUSED_PARAMETER(nVal); + zName = (const char*)sqlite3_value_text(apVal[0]); + + /* Temporarily disable the ".log" when calling shellFakeSchema() because + ** shellFakeSchema() might generate failures for some ephemeral virtual + ** tables due to missing arguments. Example: fts4aux. + ** https://sqlite.org/forum/forumpost/42fe6520b803be51 */ + p->pLog = 0; + zFake = zName? shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName) : 0; + p->pLog = pSavedLog; + + if( zFake ){ + sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), + -1, sqlite3_free); + free(zFake); + } +} + /* Flags for open_db(). ** ** The default behavior of open_db() is to exit(1) if the database fails to @@ -5853,7 +5862,7 @@ static void open_db(ShellState *p, int openFlags){ shellDtostr, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); - sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, + sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, p, shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); From 391f70c527e3e4ff05ffed90faf5dbeb303dfe46 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 1 May 2025 16:07:52 +0000 Subject: [PATCH 28/42] Allow sqlite3_rsync to work on non-WAL-mode databases, as long as the --wal-only flag is not used. FossilOrigin-Name: e4126dcd1eba4f040a7c07102d34692287b74b41a3437a3b9d15c4f8c9d4e6fd --- manifest | 16 +++++++++------ manifest.uuid | 2 +- tool/sqlite3_rsync.c | 48 ++++++++++++++++++++++++++++---------------- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index d5ac399b08..70e3a50cf1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\sproblem\sin\sthe\sCLI\sin\swhich\sSQL\serrors\sthat\soccur\sduring\nthe\s".schema"\scommand\sare\sproperly\signored,\syes\sstill\sappear\sin\sthe\s".log"\noutput.\s[forum:/forumpost/42fe6520b803be51|Forum\spost\s42fe6520b8] -D 2025-04-30T14:37:00.121 +C Allow\ssqlite3_rsync\sto\swork\son\snon-WAL-mode\sdatabases,\sas\slong\sas\sthe\n--wal-only\sflag\sis\snot\sused. +D 2025-05-01T16:07:52.716 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c 9a1cca2ab1271c59b37a6493c15dc1bcd0ab9149197a9125926bc08dd26b83fb +F tool/sqlite3_rsync.c f5e0b13cb159638a8556614dc13c9c3d457df69a7c426b9977fc3fb79130ce2d F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,8 +2207,12 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cdef486e212fe4b26605065d9cff08f608cb80df48ee64e4be63637769bdfacc -R ff4dbf6198636e450976250aaa2d8234 +P 20abf1ec107f942e4527901685d61283c9c2fe7bcefad63dbf5c6cbf050da849 +Q +9f88f73ce39e761810110dc158ec0e5a84401a80b247a5395a906a6a538d5fec +R eaf25ba59d2c9244d7aeaa66e676c510 +T *branch * rsync-non-wal-mode +T *sym-rsync-non-wal-mode * +T -sym-trunk * U drh -Z 88680290faafcb12b4513cbfd1fe3d4f +Z db3c00b4ead05d580221bc820eec3ab1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3d3fb0bd6f..c2fd98b0c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20abf1ec107f942e4527901685d61283c9c2fe7bcefad63dbf5c6cbf050da849 +e4126dcd1eba4f040a7c07102d34692287b74b41a3437a3b9d15c4f8c9d4e6fd diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index 01dcbed050..bd42a2b8c2 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -35,6 +35,7 @@ static const char zUsage[] = " --ssh PATH Name of the SSH program used to reach the remote side\n" " -v Verbose. Multiple v's for increasing output\n" " --version Show detailed version information\n" + " --wal-only Do not sync unless both databases are in WAL mode\n" ; typedef unsigned char u8; @@ -57,6 +58,7 @@ struct SQLiteRsync { u8 isReplica; /* True if running on the replica side */ u8 iProtocol; /* Protocol version number */ u8 wrongEncoding; /* ATTACH failed due to wrong encoding */ + u8 bWalOnly; /* Require WAL mode */ sqlite3_uint64 nOut; /* Bytes transmitted */ sqlite3_uint64 nIn; /* Bytes received */ unsigned int nPage; /* Total number of pages in the database */ @@ -1237,9 +1239,11 @@ static void originSide(SQLiteRsync *p){ } hashRegister(p->db); runSql(p, "BEGIN"); - runSqlReturnText(p, buf, "PRAGMA journal_mode"); - if( sqlite3_stricmp(buf,"wal")!=0 ){ - reportError(p, "Origin database is not in WAL mode"); + if( p->bWalOnly ){ + runSqlReturnText(p, buf, "PRAGMA journal_mode"); + if( sqlite3_stricmp(buf,"wal")!=0 ){ + reportError(p, "Origin database is not in WAL mode"); + } } runSqlReturnUInt(p, &nPage, "PRAGMA page_count"); runSqlReturnUInt(p, &szPg, "PRAGMA page_size"); @@ -1463,10 +1467,12 @@ static void replicaSide(SQLiteRsync *p){ runSql(p, "SELECT * FROM replica.sqlite_schema"); } runSql(p, "BEGIN IMMEDIATE"); - runSqlReturnText(p, buf, "PRAGMA replica.journal_mode"); - if( strcmp(buf, "wal")!=0 ){ - reportError(p, "replica is not in WAL mode"); - break; + if( p->bWalOnly ){ + runSqlReturnText(p, buf, "PRAGMA replica.journal_mode"); + if( strcmp(buf, "wal")!=0 ){ + reportError(p, "replica is not in WAL mode"); + break; + } } runSqlReturnUInt(p, &nRPage, "PRAGMA replica.page_count"); runSqlReturnUInt(p, &szRPage, "PRAGMA replica.page_size"); @@ -1669,11 +1675,12 @@ int main(int argc, char const * const *argv){ memset(&ctx, 0, sizeof(ctx)); for(i=1; i Date: Fri, 2 May 2025 11:18:09 +0000 Subject: [PATCH 29/42] Do not allow sqlite3_rsync to convert the replica from WAL-mode into DELETE-mode, as that can disrupt existing clients on the replica side. DELETE-mode to WAL-mode conversions are allowed, however. See [forum:/forumpost/6b575b66156673ee|forum thread 6b575b66156]. FossilOrigin-Name: 660a035b6ce6684d429b882133e032181cc1664f4efadf1bc0e4ae27d45071c4 --- manifest | 13 ++++++------- manifest.uuid | 2 +- tool/sqlite3_rsync.c | 16 ++++++++++++---- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 562708e456..96f52aa1fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\ssqlite3_rsync\sso\sthat,\sby\sdefault,\sit\swill\ssync\snon-WAL-mode\ndatabase\sfiles.\s\sAdd\sa\snew\scommand-line\soption\s--wal-only\sthat\srestricts\nthe\ssync\sto\sWAL-mode\sdatabases\sonly\s(the\sformer\sdefault).\s\sImprove\ncommand-line\soption\sparsing\sso\sthat\sonly\sa\ssingle\s"-"\sis\srequired\sbefore\neach\soption. -D 2025-05-01T18:07:27.165 +C Do\snot\sallow\ssqlite3_rsync\sto\sconvert\sthe\sreplica\sfrom\sWAL-mode\sinto\nDELETE-mode,\sas\sthat\scan\sdisrupt\sexisting\sclients\son\sthe\sreplica\sside.\nDELETE-mode\sto\sWAL-mode\sconversions\sare\sallowed,\showever.\s\sSee\n[forum:/forumpost/6b575b66156673ee|forum\sthread\s6b575b66156]. +D 2025-05-02T11:18:09.023 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c f5e0b13cb159638a8556614dc13c9c3d457df69a7c426b9977fc3fb79130ce2d +F tool/sqlite3_rsync.c a8e1962d9e0418b37d6865e483640c49498efe64bf542022e845b056f6eb9cce F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,9 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 20abf1ec107f942e4527901685d61283c9c2fe7bcefad63dbf5c6cbf050da849 e4126dcd1eba4f040a7c07102d34692287b74b41a3437a3b9d15c4f8c9d4e6fd -R eaf25ba59d2c9244d7aeaa66e676c510 -T +closed e4126dcd1eba4f040a7c07102d34692287b74b41a3437a3b9d15c4f8c9d4e6fd +P 4b53603fe468c0c28b818762917e41bdd870de6d4cc143688f1cdea3136c81a4 +R 77c0a71cca6be2a9f28e992f31255bba U drh -Z 308dcdc3a40ce25db24c1b710b31692f +Z 4ae06cab268d08247b608c21e7458e1c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ccb7909e8e..0d00d48c76 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b53603fe468c0c28b818762917e41bdd870de6d4cc143688f1cdea3136c81a4 +660a035b6ce6684d429b882133e032181cc1664f4efadf1bc0e4ae27d45071c4 diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index bd42a2b8c2..34faaf0fd1 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -1393,6 +1393,7 @@ static void replicaSide(SQLiteRsync *p){ int c; sqlite3_stmt *pIns = 0; unsigned int szOPage = 0; + char eJMode = 0; /* Journal mode prior to sync */ char buf[65536]; p->isReplica = 1; @@ -1463,16 +1464,18 @@ static void replicaSide(SQLiteRsync *p){ } if( nRPage==0 ){ runSql(p, "PRAGMA replica.page_size=%u", szOPage); - runSql(p, "PRAGMA replica.journal_mode=WAL"); runSql(p, "SELECT * FROM replica.sqlite_schema"); } runSql(p, "BEGIN IMMEDIATE"); - if( p->bWalOnly ){ - runSqlReturnText(p, buf, "PRAGMA replica.journal_mode"); - if( strcmp(buf, "wal")!=0 ){ + runSqlReturnText(p, buf, "PRAGMA replica.journal_mode"); + if( strcmp(buf, "wal")!=0 ){ + if( p->bWalOnly && nRPage>0 ){ reportError(p, "replica is not in WAL mode"); break; } + eJMode = 1; /* Non-WAL mode prior to sync */ + }else{ + eJMode = 2; /* WAL-mode prior to sync */ } runSqlReturnUInt(p, &nRPage, "PRAGMA replica.page_count"); runSqlReturnUInt(p, &szRPage, "PRAGMA replica.page_size"); @@ -1537,6 +1540,11 @@ static void replicaSide(SQLiteRsync *p){ } readBytes(p, szOPage, buf); if( p->nErr ) break; + if( pgno==1 && eJMode==2 && buf[18]==1 ){ + /* Do not switch the replica out of WAL mode if it started in + ** WAL mode */ + buf[18] = buf[19] = 2; + } p->nPageSent++; sqlite3_bind_int64(pIns, 1, pgno); sqlite3_bind_blob(pIns, 2, buf, szOPage, SQLITE_STATIC); From fbafaa561906e6c4ec38d225f33f33e3080782ad Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 2 May 2025 17:39:21 +0000 Subject: [PATCH 30/42] This is the start of an experiment in getting sqlite3_rsync to use less bandwidth when the two databases are very similar, by sending hashes over blocks of pages initially, rather than over individual pages, then requesting more detail when hashes do not match. FossilOrigin-Name: 266b4b8f0104bd4b1cff87ed78b0223006bf661a9650294a2b330d50c7ee8a0c --- manifest | 15 +- manifest.uuid | 2 +- tool/sqlite3_rsync.c | 378 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 315 insertions(+), 80 deletions(-) diff --git a/manifest b/manifest index 96f52aa1fd..05160f3b5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\ssqlite3_rsync\sto\sconvert\sthe\sreplica\sfrom\sWAL-mode\sinto\nDELETE-mode,\sas\sthat\scan\sdisrupt\sexisting\sclients\son\sthe\sreplica\sside.\nDELETE-mode\sto\sWAL-mode\sconversions\sare\sallowed,\showever.\s\sSee\n[forum:/forumpost/6b575b66156673ee|forum\sthread\s6b575b66156]. -D 2025-05-02T11:18:09.023 +C This\sis\sthe\sstart\sof\san\sexperiment\sin\sgetting\ssqlite3_rsync\sto\suse\sless\nbandwidth\swhen\sthe\stwo\sdatabases\sare\svery\ssimilar,\sby\ssending\shashes\nover\sblocks\sof\spages\sinitially,\srather\sthan\sover\sindividual\spages,\sthen\nrequesting\smore\sdetail\swhen\shashes\sdo\snot\smatch. +D 2025-05-02T17:39:21.504 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c a8e1962d9e0418b37d6865e483640c49498efe64bf542022e845b056f6eb9cce +F tool/sqlite3_rsync.c 438ffaf829181863ea941c8e6d5faabfe9161d7a464b3dce1857ac2fabe09273 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,8 +2207,11 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4b53603fe468c0c28b818762917e41bdd870de6d4cc143688f1cdea3136c81a4 -R 77c0a71cca6be2a9f28e992f31255bba +P 660a035b6ce6684d429b882133e032181cc1664f4efadf1bc0e4ae27d45071c4 +R 2f18aca9a98c725fdd5da1ebbb1e7566 +T *branch * faster-rsync +T *sym-faster-rsync * +T -sym-trunk * U drh -Z 4ae06cab268d08247b608c21e7458e1c +Z 08dacd140b0440acc75d69379c13dfa6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0d00d48c76..9d75464a6e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -660a035b6ce6684d429b882133e032181cc1664f4efadf1bc0e4ae27d45071c4 +266b4b8f0104bd4b1cff87ed78b0223006bf661a9650294a2b330d50c7ee8a0c diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index 34faaf0fd1..6fb3650949 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -75,20 +75,26 @@ struct SQLiteRsync { /* Magic numbers to identify particular messages sent over the wire. */ +/**** Baseline: protocol version 1 ****/ #define ORIGIN_BEGIN 0x41 /* Initial message */ #define ORIGIN_END 0x42 /* Time to quit */ #define ORIGIN_ERROR 0x43 /* Error message from the remote */ #define ORIGIN_PAGE 0x44 /* New page data */ #define ORIGIN_TXN 0x45 /* Transaction commit */ #define ORIGIN_MSG 0x46 /* Informational message */ +/**** Added in protocol version 2 ****/ +#define ORIGIN_DETAIL 0x47 /* Request finer-grain hash info */ +#define ORIGIN_READY 0x48 /* Ready for next round of hash exchanges */ +/**** Baseline: protocol version 1 ****/ #define REPLICA_BEGIN 0x61 /* Welcome message */ #define REPLICA_ERROR 0x62 /* Error. Report and quit. */ #define REPLICA_END 0x63 /* Replica wants to stop */ #define REPLICA_HASH 0x64 /* One or more pages hashes to report */ #define REPLICA_READY 0x65 /* Read to receive page content */ #define REPLICA_MSG 0x66 /* Informational message */ - +/**** Added in protocol version 2 ****/ +#define REPLICA_CONFIG 0x67 /* Hash exchange configuration */ /**************************************************************************** ** Beginning of the popen2() implementation copied from Fossil ************* @@ -796,11 +802,49 @@ static void hashFunc( sqlite3_result_blob(context, HashFinal(&cx), 160/8, SQLITE_TRANSIENT); } +/* +** Implementation of the agghash(X) function. +** +** Return a 160-bit BLOB which is the hash of the concatenation +** of all X inputs. +*/ +static void agghashStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + HashContext *pCx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + if( eType==SQLITE_NULL ) return; + pCx = (HashContext*)sqlite3_aggregate_context(context, sizeof(*pCx)); + if( pCx==0 ) return; + if( pCx->iSize==0 ) HashInit(pCx, 160); + if( eType==SQLITE_BLOB ){ + HashUpdate(pCx, sqlite3_value_blob(argv[0]), nByte); + }else{ + HashUpdate(pCx, sqlite3_value_text(argv[0]), nByte); + } +} +static void agghashFinal(sqlite3_context *context){ + HashContext *pCx = (HashContext*)sqlite3_aggregate_context(context, 0); + if( pCx ){ + sqlite3_result_blob(context, HashFinal(pCx), 160/8, SQLITE_TRANSIENT); + } +} + /* Register the hash function */ static int hashRegister(sqlite3 *db){ - return sqlite3_create_function(db, "hash", 1, + int rc; + rc = sqlite3_create_function(db, "hash", 1, SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, 0, hashFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "agghash", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, 0, agghashStep, agghashFinal); + } + return rc; } /* End of the hashing logic @@ -1192,6 +1236,13 @@ static void closeDb(SQLiteRsync *p){ ** nPage, and szPage. Then enter a loop responding to message from ** the replica: ** +** REPLICA_BEGIN iProtocol +** +** An optional message sent by the replica in response to the +** prior ORIGIN_BEGIN with a counter-proposal for the protocol +** level. If seen, try to reduce the protocol level to what is +** requested and send a new ORGIN_BEGIN. +** ** REPLICA_ERROR size text ** ** Report an error from the replica and quit @@ -1202,24 +1253,36 @@ static void closeDb(SQLiteRsync *p){ ** ** REPLICA_HASH hash ** -** The argument is the 20-byte SHA1 hash for the next page -** page hashes appear in sequential order with no gaps. +** The argument is the 20-byte SHA1 hash for the next page or +** block of pages. Hashes appear in sequential order with no gaps, +** unless there is an intervening REPLICA_CONFIG message. +** +** REPLICA_CONFIG pgno cnt +** +** Set counters used by REPLICA_HASH. The next hash will start +** on page pgno and all subsequent hashes will cover cnt pages +** each. Note that for a multi-page hash, the hash value is +** actually a hash of the individual page hashes. ** ** REPLICA_READY ** ** The replica has sent all the hashes that it intends to send. ** This side (the origin) can now start responding with page -** content for pages that do not have a matching hash. +** content for pages that do not have a matching hash or with +** ORIGIN_DETAIL messages with requests for more detail. */ static void originSide(SQLiteRsync *p){ int rc = 0; int c = 0; unsigned int nPage = 0; - unsigned int iPage = 0; + unsigned int iHash = 1; /* Pgno for next hash to receive */ + unsigned int nHash = 1; /* Number of pages per hash received */ unsigned int lockBytePage = 0; unsigned int szPg = 0; - sqlite3_stmt *pCkHash = 0; - sqlite3_stmt *pInsHash = 0; + sqlite3_stmt *pCkHash = 0; /* Verify hash on a single page */ + sqlite3_stmt *pCkHashN = 0; /* Verify a multi-page hash */ + sqlite3_stmt *pInsHash = 0; /* Record a bad hash */ + unsigned int nMulti = 0; /* Multi-page hashes not matched */ char buf[200]; p->isReplica = 0; @@ -1270,11 +1333,16 @@ static void originSide(SQLiteRsync *p){ ** that is larger than what it knows about. The replica sends back ** a counter-proposal of an earlier protocol which the origin can ** accept by resending a new ORIGIN_BEGIN. */ - p->iProtocol = readByte(p); - writeByte(p, ORIGIN_BEGIN); - writeByte(p, p->iProtocol); - writePow2(p, p->szPage); - writeUint32(p, p->nPage); + u8 newProtocol = readByte(p); + if( newProtocol < p->iProtocol ){ + p->iProtocol = newProtocol; + writeByte(p, ORIGIN_BEGIN); + writeByte(p, p->iProtocol); + writePow2(p, p->szPage); + writeUint32(p, p->nPage); + }else{ + reportError(p, "Invalid REPLICA_BEGIN reply"); + } break; } case REPLICA_MSG: @@ -1282,25 +1350,60 @@ static void originSide(SQLiteRsync *p){ readAndDisplayMessage(p, c); break; } + case REPLICA_CONFIG: { + readUint32(p, &iHash); + readUint32(p, &nHash); + break; + } case REPLICA_HASH: { if( pCkHash==0 ){ - runSql(p, "CREATE TEMP TABLE badHash(pgno INTEGER PRIMARY KEY)"); + runSql(p, "CREATE TEMP TABLE badHash(" + " pgno INTEGER PRIMARY KEY," + " sz INT)"); pCkHash = prepareStmt(p, "SELECT pgno FROM sqlite_dbpage('main')" - " WHERE pgno=?1 AND hash(data)!=?2" + " WHERE pgno=?1 AND hash(data)!=?3" ); if( pCkHash==0 ) break; - pInsHash = prepareStmt(p, "INSERT INTO badHash VALUES(?)"); + pInsHash = prepareStmt(p, "INSERT INTO badHash VALUES(?1,?2)"); if( pInsHash==0 ) break; } p->nHashSent++; - iPage++; - sqlite3_bind_int64(pCkHash, 1, iPage); - readBytes(p, 20, buf); - sqlite3_bind_blob(pCkHash, 2, buf, 20, SQLITE_STATIC); - rc = sqlite3_step(pCkHash); + if( nHash>1 ){ + if( pCkHashN==0 ){ + pCkHashN = prepareStmt(p, + "WITH a1(pgno) AS " + "(VALUES(?1) UNION ALL SELECT pgno+1 FROM a1 WHERE pgnodb)); + } + sqlite3_reset(pCkHashN); + }else{ + sqlite3_bind_int64(pCkHash, 1, iHash); + readBytes(p, 20, buf); + sqlite3_bind_blob(pCkHash, 3, buf, 20, SQLITE_STATIC); + rc = sqlite3_step(pCkHash); + if( rc==SQLITE_ERROR ){ + reportError(p, "SQL statement [%s] failed: %s", + sqlite3_sql(pCkHash), sqlite3_errmsg(p->db)); + } + sqlite3_reset(pCkHash); + } if( rc==SQLITE_ROW ){ - sqlite3_bind_int64(pInsHash, 1, sqlite3_column_int64(pCkHash, 0)); + sqlite3_bind_int64(pInsHash, 1, iHash); + sqlite3_bind_int64(pInsHash, 2, nHash); rc = sqlite3_step(pInsHash); if( rc!=SQLITE_DONE ){ reportError(p, "SQL statement [%s] failed: %s", @@ -1308,42 +1411,57 @@ static void originSide(SQLiteRsync *p){ } sqlite3_reset(pInsHash); } - else if( rc!=SQLITE_DONE ){ - reportError(p, "SQL statement [%s] failed: %s", - sqlite3_sql(pCkHash), sqlite3_errmsg(p->db)); - } - sqlite3_reset(pCkHash); + iHash += nHash; break; } case REPLICA_READY: { - sqlite3_stmt *pStmt; - sqlite3_finalize(pCkHash); - sqlite3_finalize(pInsHash); - pCkHash = 0; - pInsHash = 0; - if( iPage+1nPage ){ - runSql(p, "WITH RECURSIVE c(n) AS" - " (VALUES(%d) UNION ALL SELECT n+1 FROM c WHERE n<%d)" - " INSERT INTO badHash SELECT n FROM c", - iPage+1, p->nPage); + if( nMulti>0 ){ + sqlite3_stmt *pStmt; + pStmt = prepareStmt(p,"SELECT pgno, sz FROM badHash WHERE sz>1"); + if( pStmt==0 ) break; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + writeByte(p, ORIGIN_DETAIL); + writeUint32(p, sqlite3_column_int(pStmt, 0)); + writeUint32(p, sqlite3_column_int(pStmt, 1)); + } + sqlite3_finalize(pStmt); + runSql(p, "DELETE FROM badHash WHERE sz>1"); + nMulti = 0; + writeByte(p, ORIGIN_READY); + }else{ + sqlite3_stmt *pStmt; + sqlite3_finalize(pCkHash); + sqlite3_finalize(pCkHashN); + sqlite3_finalize(pInsHash); + pCkHash = 0; + pInsHash = 0; + if( iHash+1nPage ){ + runSql(p, "WITH RECURSIVE c(n) AS" + " (VALUES(%d) UNION ALL SELECT n+1 FROM c WHERE n<%d)" + " INSERT INTO badHash SELECT n, 1 FROM c", + iHash+1, p->nPage); + } + runSql(p, "DELETE FROM badHash WHERE pgno=%d", lockBytePage); + pStmt = prepareStmt(p, + "SELECT pgno, data" + " FROM badHash JOIN sqlite_dbpage('main') USING(pgno)"); + if( pStmt==0 ) break; + while( sqlite3_step(pStmt)==SQLITE_ROW + && p->nErr==0 + && p->nWrErr==0 + ){ + unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0); + const void *pContent = sqlite3_column_blob(pStmt, 1); + writeByte(p, ORIGIN_PAGE); + writeUint32(p, pgno); + writeBytes(p, szPg, pContent); + p->nPageSent++; + } + sqlite3_finalize(pStmt); + writeByte(p, ORIGIN_TXN); + writeUint32(p, nPage); + writeByte(p, ORIGIN_END); } - runSql(p, "DELETE FROM badHash WHERE pgno=%d", lockBytePage); - pStmt = prepareStmt(p, - "SELECT pgno, data" - " FROM badHash JOIN sqlite_dbpage('main') USING(pgno)"); - if( pStmt==0 ) break; - while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 && p->nWrErr==0 ){ - unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0); - const void *pContent = sqlite3_column_blob(pStmt, 1); - writeByte(p, ORIGIN_PAGE); - writeUint32(p, pgno); - writeBytes(p, szPg, pContent); - p->nPageSent++; - } - sqlite3_finalize(pStmt); - writeByte(p, ORIGIN_TXN); - writeUint32(p, nPage); - writeByte(p, ORIGIN_END); fflush(p->pOut); break; } @@ -1360,6 +1478,88 @@ static void originSide(SQLiteRsync *p){ closeDb(p); } +/* +** Send a REPLICA_HASH message for each entry in the sendHash table. +** The sendHash table looks like this: +** +** CREATE TABLE sendHash( +** fpg INTEGER PRIMARY KEY, -- Page number of the hash +** npg INT -- Number of pages in this hash +** ); +** +** If iHash is page number for the next page that the origin will +** be expecting, and nHash is the number of pages that the origin will +** be expecting in the hash that follows. Send a REPLICA_CONFIG message +** if either of these values if not correct. +*/ +static void sendHashMessages( + SQLiteRsync *p, /* The replica-side of the sync */ + unsigned int iHash, /* Next page expected by origin */ + unsigned int nHash /* Next number of pages expected by origin */ +){ + sqlite3_stmt *pStmt; + pStmt = prepareStmt(p, + "SELECT if(npg==1," + " (SELECT hash(data) FROM sqlite_dbpage('replica') WHERE pgno=fpg)," + " (WITH RECURSIVE c(n) AS" + " (SELECT fpg UNION ALL SELECT n+1 FROM c WHERE nnErr==0 && p->nWrErr==0 ){ + const unsigned char *a = sqlite3_column_blob(pStmt, 0); + unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt, 1); + unsigned int npg = (unsigned int)sqlite3_column_int64(pStmt, 2); + if( pgno!=iHash || npg!=nHash ){ + writeByte(p, REPLICA_CONFIG); + writeUint32(p, pgno); + writeUint32(p, npg); + } + writeByte(p, REPLICA_HASH); + writeBytes(p, 20, a); + p->nHashSent++; + iHash = pgno + npg; + nHash = npg; + } + sqlite3_finalize(pStmt); + runSql(p, "DELETE FROM sendHash"); + writeByte(p, REPLICA_READY); + fflush(p->pOut); +} + +/* +** Make entries in the sendHash table to send hashes for +** npg (mnemonic: Number of PaGes) pages starting with fpg +** (mnemonic: First PaGe). +*/ +static void subdivideHashRange( + SQLiteRsync *p, /* The replica-side of the sync */ + unsigned int fpg, /* First page of the range */ + unsigned int npg /* Number of pages */ +){ + unsigned int nChunk; /* How many pages to request per hash */ + sqlite3_uint64 iEnd; /* One more than the last page */ + if( npg<=30 ){ + nChunk = 1; + }else if( npg<=1000 ){ + nChunk = 30; + }else{ + nChunk = 1000; + } + iEnd = fpg; + iEnd += npg; + runSql(p, + "WITH RECURSIVE c(n) AS" + " (VALUES(%u) UNION ALL SELECT n+%u FROM c WHERE n<%llu)" + "REPLACE INTO sendHash(fpg,npg)" + " SELECT n, min(%llu-fpg,%u) FROM c", + fpg, nChunk, iEnd, iEnd, nChunk + ); +} + /* ** Run the replica-side protocol. The protocol is passive in the sense ** that it only response to message from the origin side. @@ -1370,15 +1570,35 @@ static void originSide(SQLiteRsync *p){ ** each page in the origin database (sent as a single-byte power-of-2), ** and the number of pages in the origin database. ** This procedure checks compatibility, and if everything is ok, -** it starts sending hashes of pages already present back to the origin. +** it starts sending hashes back to the origin using REPLICA_HASH +** and/or REPLICA_CONFIG message, followed by a single REPLICA_READY. +** REPLICA_CONFIG is only sent if the protocol is 2 or greater. ** -** ORIGIN_ERROR size text +** ORIGIN_ERROR size text ** -** Report the received error and quit. +** Report an error and quit. ** -** ORIGIN_PAGE pgno content +** ORIGIN_DETAIL pgno cnt ** -** Update the content of the given page. +** The origin reports that a multi-page hash starting at pgno and +** spanning cnt pages failed to match. The origin is requesting +** details (more REPLICA_HASH message with a smaller cnt). The +** replica must wait on ORIGIN_READY before sending its reply. +** +** ORIGIN_READY +** +** After sending one or more ORIGIN_DETAIL messages, the ORIGIN_READY +** is sent by the origin to indicate that it has finished sending +** requests for detail and is ready for the replicate to reply +** with a new round of REPLICA_CONFIG and REPLICA_HASH messages. +** +** ORIGIN_PAGE pgno content +** +** Once the origin believes it knows exactly which pages need to be +** updated in the replica, it starts sending those pages using these +** messages. These messages will only appear immediately after +** REPLICA_READY. The origin never mixes ORIGIN_DETAIL and +** ORIGIN_PAGE messages in the same batch. ** ** ORIGIN_TXN pgno ** @@ -1418,7 +1638,6 @@ static void replicaSide(SQLiteRsync *p){ unsigned int nOPage = 0; unsigned int nRPage = 0, szRPage = 0; int rc = 0; - sqlite3_stmt *pStmt = 0; closeDb(p); p->iProtocol = readByte(p); @@ -1458,6 +1677,12 @@ static void replicaSide(SQLiteRsync *p){ closeDb(p); break; } + runSql(p, + "CREATE TABLE sendHash(" + " fpg INTEGER PRIMARY KEY," /* The page number of hash to send */ + " npg INT" /* Number of pages in this hash */ + ")" + ); hashRegister(p->db); if( runSqlReturnUInt(p, &nRPage, "PRAGMA replica.page_count") ){ break; @@ -1484,23 +1709,30 @@ static void replicaSide(SQLiteRsync *p){ "replica is %d bytes", szOPage, szRPage); break; } - - pStmt = prepareStmt(p, - "SELECT hash(data) FROM sqlite_dbpage('replica')" - " WHERE pgno<=min(%d,%d)" - " ORDER BY pgno", nRPage, nOPage); - while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 && p->nWrErr==0 ){ - const unsigned char *a = sqlite3_column_blob(pStmt, 0); - writeByte(p, REPLICA_HASH); - writeBytes(p, 20, a); - p->nHashSent++; + if( p->iProtocol<2 ){ + runSql(p, + "WITH RECURSIVE c(n) AS" + "(VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<%d)" + "INSERT INTO sendHash(fpg, npg) SELECT n, 1 FROM c", + nRPage); + }else{ + subdivideHashRange(p, 1, nRPage); } - sqlite3_finalize(pStmt); - writeByte(p, REPLICA_READY); - fflush(p->pOut); + sendHashMessages(p, 1, 1); runSql(p, "PRAGMA writable_schema=ON"); break; } + case ORIGIN_DETAIL: { + unsigned int fpg, npg; + readUint32(p, &fpg); + readUint32(p, &npg); + subdivideHashRange(p, fpg, npg); + break; + } + case ORIGIN_READY: { + sendHashMessages(p, 0, 0); + break; + } case ORIGIN_TXN: { unsigned int nOPage = 0; readUint32(p, &nOPage); From f124ddf36a7945ce9048f15cd7d83047d424fb16 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 2 May 2025 18:32:46 +0000 Subject: [PATCH 31/42] Bug fixes. Added new debugging features to better visualize the protocol. FossilOrigin-Name: c70330668690e7c3c55ae34137d5b2c91871432004b82b2b23a89fc3f1322a62 --- manifest | 15 +++--- manifest.uuid | 2 +- tool/sqlite3_rsync.c | 113 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 113 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 05160f3b5a..d61a0efa66 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sis\sthe\sstart\sof\san\sexperiment\sin\sgetting\ssqlite3_rsync\sto\suse\sless\nbandwidth\swhen\sthe\stwo\sdatabases\sare\svery\ssimilar,\sby\ssending\shashes\nover\sblocks\sof\spages\sinitially,\srather\sthan\sover\sindividual\spages,\sthen\nrequesting\smore\sdetail\swhen\shashes\sdo\snot\smatch. -D 2025-05-02T17:39:21.504 +C Bug\sfixes.\s\sAdded\snew\sdebugging\sfeatures\sto\sbetter\svisualize\sthe\nprotocol. +D 2025-05-02T18:32:46.784 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c 438ffaf829181863ea941c8e6d5faabfe9161d7a464b3dce1857ac2fabe09273 +F tool/sqlite3_rsync.c b786fe6316fe7ab4dc9c9f4f81c202f1baac66a49ef7a198121472f684977ce9 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,11 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 660a035b6ce6684d429b882133e032181cc1664f4efadf1bc0e4ae27d45071c4 -R 2f18aca9a98c725fdd5da1ebbb1e7566 -T *branch * faster-rsync -T *sym-faster-rsync * -T -sym-trunk * +P 266b4b8f0104bd4b1cff87ed78b0223006bf661a9650294a2b330d50c7ee8a0c +R b7a2f023d4216a23fb40df2130b3b81c U drh -Z 08dacd140b0440acc75d69379c13dfa6 +Z 82daa27df570baf781be22c975da80d8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9d75464a6e..4ea32c916d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -266b4b8f0104bd4b1cff87ed78b0223006bf661a9650294a2b330d50c7ee8a0c +c70330668690e7c3c55ae34137d5b2c91871432004b82b2b23a89fc3f1322a62 diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index 6fb3650949..bf8a8dfb99 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -46,9 +46,11 @@ struct SQLiteRsync { const char *zOrigin; /* Name of the origin */ const char *zReplica; /* Name of the replica */ const char *zErrFile; /* Append error messages to this file */ + const char *zDebugFile; /* Append debugging messages to this file */ FILE *pOut; /* Transmit to the other side */ FILE *pIn; /* Receive from the other side */ FILE *pLog; /* Duplicate output here if not NULL */ + FILE *pDebug; /* Write debug info here if not NULL */ sqlite3 *db; /* Database connection */ int nErr; /* Number of errors encountered */ int nWrErr; /* Number of failed attempts to write on the pipe */ @@ -70,7 +72,7 @@ struct SQLiteRsync { /* The version number of the protocol. Sent in the *_BEGIN message ** to verify that both sides speak the same dialect. */ -#define PROTOCOL_VERSION 1 +#define PROTOCOL_VERSION 2 /* Magic numbers to identify particular messages sent over the wire. @@ -882,6 +884,25 @@ static void logError(SQLiteRsync *p, const char *zFormat, ...){ p->nErr++; } +/* +** Append text to the debugging mesage file, if an that file is +** specified. +*/ +static void debugMessage(SQLiteRsync *p, const char *zFormat, ...){ + if( p->zDebugFile ){ + if( p->pDebug==0 ){ + p->pDebug = fopen(p->zDebugFile, "wb"); + } + if( p->pDebug ){ + va_list ap; + va_start(ap, zFormat); + vfprintf(p->pDebug, zFormat, ap); + va_end(ap); + fflush(p->pDebug); + } + } +} + /* Read a single big-endian 32-bit unsigned integer from the input ** stream. Return 0 on success and 1 if there are any errors. @@ -1334,12 +1355,19 @@ static void originSide(SQLiteRsync *p){ ** a counter-proposal of an earlier protocol which the origin can ** accept by resending a new ORIGIN_BEGIN. */ u8 newProtocol = readByte(p); + if( p->zDebugFile ){ + debugMessage(p, "<- REPLICA_BEGIN %d\n", (int)newProtocol); + } if( newProtocol < p->iProtocol ){ p->iProtocol = newProtocol; writeByte(p, ORIGIN_BEGIN); writeByte(p, p->iProtocol); writePow2(p, p->szPage); writeUint32(p, p->nPage); + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_BEGIN %d %d %u\n", p->iProtocol, + p->szPage, p->nPage); + } }else{ reportError(p, "Invalid REPLICA_BEGIN reply"); } @@ -1353,6 +1381,9 @@ static void originSide(SQLiteRsync *p){ case REPLICA_CONFIG: { readUint32(p, &iHash); readUint32(p, &nHash); + if( p->zDebugFile ){ + debugMessage(p, "<- REPLICA_CONFIG %u %u\n", iHash, nHash); + } break; } case REPLICA_HASH: { @@ -1369,14 +1400,17 @@ static void originSide(SQLiteRsync *p){ if( pInsHash==0 ) break; } p->nHashSent++; + if( p->zDebugFile ){ + debugMessage(p, "<- REPLICA_HASH %u %u\n", iHash, nHash); + } if( nHash>1 ){ if( pCkHashN==0 ){ pCkHashN = prepareStmt(p, "WITH a1(pgno) AS " "(VALUES(?1) UNION ALL SELECT pgno+1 FROM a1 WHERE pgnozDebugFile ){ + debugMessage(p, "<- REPLICA_READY\n"); + } if( nMulti>0 ){ sqlite3_stmt *pStmt; pStmt = prepareStmt(p,"SELECT pgno, sz FROM badHash WHERE sz>1"); if( pStmt==0 ) break; while( sqlite3_step(pStmt)==SQLITE_ROW ){ + unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0); + unsigned int cnt = (unsigned int)sqlite3_column_int64(pStmt,1); writeByte(p, ORIGIN_DETAIL); - writeUint32(p, sqlite3_column_int(pStmt, 0)); - writeUint32(p, sqlite3_column_int(pStmt, 1)); + writeUint32(p, pgno); + writeUint32(p, cnt); + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_DETAIL %u %u\n", pgno, cnt); + } } sqlite3_finalize(pStmt); runSql(p, "DELETE FROM badHash WHERE sz>1"); nMulti = 0; writeByte(p, ORIGIN_READY); + if( p->zDebugFile ) debugMessage(p, "-> ORIGIN_READY\n"); }else{ sqlite3_stmt *pStmt; sqlite3_finalize(pCkHash); @@ -1456,10 +1499,16 @@ static void originSide(SQLiteRsync *p){ writeUint32(p, pgno); writeBytes(p, szPg, pContent); p->nPageSent++; + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_PAGE %u\n", pgno); + } } sqlite3_finalize(pStmt); writeByte(p, ORIGIN_TXN); writeUint32(p, nPage); + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_TXN %u\n", nPage); + } writeByte(p, ORIGIN_END); } fflush(p->pOut); @@ -1517,9 +1566,13 @@ static void sendHashMessages( writeByte(p, REPLICA_CONFIG); writeUint32(p, pgno); writeUint32(p, npg); + if( p->zDebugFile ){ + debugMessage(p, "-> REPLICA_CONFIG %u %u\n", pgno, npg); + } } writeByte(p, REPLICA_HASH); writeBytes(p, 20, a); + if( p->zDebugFile ) debugMessage(p, "-> REPLICA_HASH %u\n", iHash); p->nHashSent++; iHash = pgno + npg; nHash = npg; @@ -1528,6 +1581,7 @@ static void sendHashMessages( runSql(p, "DELETE FROM sendHash"); writeByte(p, REPLICA_READY); fflush(p->pOut); + if( p->zDebugFile ) debugMessage(p, "-> REPLICA_READY\n", iHash); } /* @@ -1555,8 +1609,8 @@ static void subdivideHashRange( "WITH RECURSIVE c(n) AS" " (VALUES(%u) UNION ALL SELECT n+%u FROM c WHERE n<%llu)" "REPLACE INTO sendHash(fpg,npg)" - " SELECT n, min(%llu-fpg,%u) FROM c", - fpg, nChunk, iEnd, iEnd, nChunk + " SELECT n, min(%llu-n,%u) FROM c", + fpg, nChunk, iEnd-nChunk, iEnd, nChunk ); } @@ -1643,6 +1697,10 @@ static void replicaSide(SQLiteRsync *p){ p->iProtocol = readByte(p); szOPage = readPow2(p); readUint32(p, &nOPage); + if( p->zDebugFile ){ + debugMessage(p, "<- ORIGIN_BEGIN %d %d %u\n", p->iProtocol, szOPage, + nOPage); + } if( p->nErr ) break; if( p->iProtocol>PROTOCOL_VERSION ){ /* If the protocol version on the origin side is larger, send back @@ -1651,6 +1709,9 @@ static void replicaSide(SQLiteRsync *p){ ** a new ORIGIN_BEGIN with a reduced protocol version. */ writeByte(p, REPLICA_BEGIN); writeByte(p, PROTOCOL_VERSION); + if( p->zDebugFile ){ + debugMessage(p, "-> REPLICA_BEGIN %u\n", PROTOCOL_VERSION); + } break; } p->nPage = nOPage; @@ -1726,16 +1787,25 @@ static void replicaSide(SQLiteRsync *p){ unsigned int fpg, npg; readUint32(p, &fpg); readUint32(p, &npg); + if( p->zDebugFile ){ + debugMessage(p, "<- ORIGIN_DETAIL %u %u\n", fpg, npg); + } subdivideHashRange(p, fpg, npg); break; } case ORIGIN_READY: { + if( p->zDebugFile ){ + debugMessage(p, "<- ORIGIN_READY\n"); + } sendHashMessages(p, 0, 0); break; } case ORIGIN_TXN: { unsigned int nOPage = 0; readUint32(p, &nOPage); + if( p->zDebugFile ){ + debugMessage(p, "<- ORIGIN_TXN %u\n", nOPage); + } if( pIns==0 ){ /* Nothing has changed */ runSql(p, "COMMIT"); @@ -1763,6 +1833,9 @@ static void replicaSide(SQLiteRsync *p){ unsigned int pgno = 0; int rc; readUint32(p, &pgno); + if( p->zDebugFile ){ + debugMessage(p, "<- ORIGIN_PAGE %u\n", pgno); + } if( p->nErr ) break; if( pIns==0 ){ pIns = prepareStmt(p, @@ -1910,6 +1983,7 @@ int main(int argc, char const * const *argv){ sqlite3_int64 tmEnd; sqlite3_int64 tmElapse; const char *zRemoteErrFile = 0; + const char *zRemoteDebugFile = 0; #define cli_opt_val cmdline_option_value(argc, argv, ++i) memset(&ctx, 0, sizeof(ctx)); @@ -1961,6 +2035,19 @@ int main(int argc, char const * const *argv){ zRemoteErrFile = cli_opt_val; continue; } + if( strcmp(z, "-debugfile")==0 ){ + /* DEBUG OPTION: --debugfile FILENAME + ** Debugging messages on the local side are written into FILENAME */ + ctx.zDebugFile = cli_opt_val; + continue; + } + if( strcmp(z, "-remote-debugfile")==0 ){ + /* DEBUG OPTION: --remote-debugfile FILENAME + ** Error messages on the remote side are written into FILENAME on + ** the remote side. */ + zRemoteDebugFile = cli_opt_val; + continue; + } if( strcmp(z, "-wal-only")==0 ){ ctx.bWalOnly = 1; continue; @@ -2070,6 +2157,10 @@ int main(int argc, char const * const *argv){ append_escaped_arg(pStr, "--errorfile", 0); append_escaped_arg(pStr, zRemoteErrFile, 1); } + if( zRemoteDebugFile ){ + append_escaped_arg(pStr, "--debugfile", 0); + append_escaped_arg(pStr, zRemoteDebugFile, 1); + } if( ctx.bWalOnly ){ append_escaped_arg(pStr, "--wal-only", 0); } @@ -2099,6 +2190,10 @@ int main(int argc, char const * const *argv){ append_escaped_arg(pStr, "--errorfile", 0); append_escaped_arg(pStr, zRemoteErrFile, 1); } + if( zRemoteDebugFile ){ + append_escaped_arg(pStr, "--debugfile", 0); + append_escaped_arg(pStr, zRemoteDebugFile, 1); + } append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1); append_escaped_arg(pStr, zDiv, 1); zCmd = sqlite3_str_finish(pStr); @@ -2120,6 +2215,10 @@ int main(int argc, char const * const *argv){ append_escaped_arg(pStr, "--errorfile", 0); append_escaped_arg(pStr, zRemoteErrFile, 1); } + if( zRemoteDebugFile ){ + append_escaped_arg(pStr, "--debugfile", 0); + append_escaped_arg(pStr, zRemoteDebugFile, 1); + } append_escaped_arg(pStr, ctx.zOrigin, 1); append_escaped_arg(pStr, ctx.zReplica, 1); zCmd = sqlite3_str_finish(pStr); From bef95654857e493dc3c644f0ade2376b307daba7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 2 May 2025 22:25:40 +0000 Subject: [PATCH 32/42] Now appears to be working. More testing needed. Refinement of the version-2 algorithm needed. FossilOrigin-Name: cb035181d9fb5909696b8ec8f9c3eeb7a7dfb4b50e82e1d3f2d5ad150afcc0ff --- manifest | 12 +++--- manifest.uuid | 2 +- tool/sqlite3_rsync.c | 99 ++++++++++++++++++++++++++------------------ 3 files changed, 66 insertions(+), 47 deletions(-) diff --git a/manifest b/manifest index d61a0efa66..3b78d8c88d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bug\sfixes.\s\sAdded\snew\sdebugging\sfeatures\sto\sbetter\svisualize\sthe\nprotocol. -D 2025-05-02T18:32:46.784 +C Now\sappears\sto\sbe\sworking.\s\sMore\stesting\sneeded.\s\sRefinement\sof\sthe\nversion-2\salgorithm\sneeded. +D 2025-05-02T22:25:40.184 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c b786fe6316fe7ab4dc9c9f4f81c202f1baac66a49ef7a198121472f684977ce9 +F tool/sqlite3_rsync.c 7f61ac7b58fe7d0d5a6abbd1570c75bce9c160fc0d5b1b679b2213d03f047b27 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 266b4b8f0104bd4b1cff87ed78b0223006bf661a9650294a2b330d50c7ee8a0c -R b7a2f023d4216a23fb40df2130b3b81c +P c70330668690e7c3c55ae34137d5b2c91871432004b82b2b23a89fc3f1322a62 +R 68278fe558d911d1ca48d9ecb46d934b U drh -Z 82daa27df570baf781be22c975da80d8 +Z 92a9f6cb386b9d8c6c4ee41cda50ebec # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4ea32c916d..c81f190ec5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c70330668690e7c3c55ae34137d5b2c91871432004b82b2b23a89fc3f1322a62 +cb035181d9fb5909696b8ec8f9c3eeb7a7dfb4b50e82e1d3f2d5ad150afcc0ff diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index bf8a8dfb99..62f751ad34 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -1298,12 +1298,12 @@ static void originSide(SQLiteRsync *p){ unsigned int nPage = 0; unsigned int iHash = 1; /* Pgno for next hash to receive */ unsigned int nHash = 1; /* Number of pages per hash received */ + unsigned int mxHash = 0; /* Maximum hash value received */ unsigned int lockBytePage = 0; unsigned int szPg = 0; sqlite3_stmt *pCkHash = 0; /* Verify hash on a single page */ sqlite3_stmt *pCkHashN = 0; /* Verify a multi-page hash */ sqlite3_stmt *pInsHash = 0; /* Record a bad hash */ - unsigned int nMulti = 0; /* Multi-page hashes not matched */ char buf[200]; p->isReplica = 0; @@ -1387,38 +1387,37 @@ static void originSide(SQLiteRsync *p){ break; } case REPLICA_HASH: { + int bMatch = 0; if( pCkHash==0 ){ runSql(p, "CREATE TEMP TABLE badHash(" " pgno INTEGER PRIMARY KEY," " sz INT)"); pCkHash = prepareStmt(p, - "SELECT pgno FROM sqlite_dbpage('main')" - " WHERE pgno=?1 AND hash(data)!=?3" + "SELECT hash(data)==?3 FROM sqlite_dbpage('main')" + " WHERE pgno=?1" ); if( pCkHash==0 ) break; pInsHash = prepareStmt(p, "INSERT INTO badHash VALUES(?1,?2)"); if( pInsHash==0 ) break; } p->nHashSent++; - if( p->zDebugFile ){ - debugMessage(p, "<- REPLICA_HASH %u %u\n", iHash, nHash); - } + readBytes(p, 20, buf); if( nHash>1 ){ if( pCkHashN==0 ){ pCkHashN = prepareStmt(p, - "WITH a1(pgno) AS " - "(VALUES(?1) UNION ALL SELECT pgno+1 FROM a1 WHERE pgnodb)); @@ -1426,16 +1425,24 @@ static void originSide(SQLiteRsync *p){ sqlite3_reset(pCkHashN); }else{ sqlite3_bind_int64(pCkHash, 1, iHash); - readBytes(p, 20, buf); sqlite3_bind_blob(pCkHash, 3, buf, 20, SQLITE_STATIC); rc = sqlite3_step(pCkHash); if( rc==SQLITE_ERROR ){ reportError(p, "SQL statement [%s] failed: %s", sqlite3_sql(pCkHash), sqlite3_errmsg(p->db)); + }else if( rc==SQLITE_ROW && sqlite3_column_int(pCkHash,0) ){ + bMatch = 1; } sqlite3_reset(pCkHash); } - if( rc==SQLITE_ROW ){ + if( p->zDebugFile ){ + debugMessage(p, "<- REPLICA_HASH %u %u %s %08x...\n", + iHash, nHash, + bMatch ? "match" : "fail", + *(unsigned int*)buf + ); + } + if( !bMatch ){ sqlite3_bind_int64(pInsHash, 1, iHash); sqlite3_bind_int64(pInsHash, 2, nHash); rc = sqlite3_step(pInsHash); @@ -1445,30 +1452,32 @@ static void originSide(SQLiteRsync *p){ } sqlite3_reset(pInsHash); } + if( iHash+nHash>mxHash ) mxHash = iHash+nHash; iHash += nHash; break; } case REPLICA_READY: { + int nMulti = 0; + sqlite3_stmt *pStmt; if( p->zDebugFile ){ debugMessage(p, "<- REPLICA_READY\n"); } - if( nMulti>0 ){ - sqlite3_stmt *pStmt; - pStmt = prepareStmt(p,"SELECT pgno, sz FROM badHash WHERE sz>1"); - if( pStmt==0 ) break; - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0); - unsigned int cnt = (unsigned int)sqlite3_column_int64(pStmt,1); - writeByte(p, ORIGIN_DETAIL); - writeUint32(p, pgno); - writeUint32(p, cnt); - if( p->zDebugFile ){ - debugMessage(p, "-> ORIGIN_DETAIL %u %u\n", pgno, cnt); - } + pStmt = prepareStmt(p,"SELECT pgno, sz FROM badHash WHERE sz>1"); + if( pStmt==0 ) break; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0); + unsigned int cnt = (unsigned int)sqlite3_column_int64(pStmt,1); + writeByte(p, ORIGIN_DETAIL); + writeUint32(p, pgno); + writeUint32(p, cnt); + nMulti++; + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_DETAIL %u %u\n", pgno, cnt); } - sqlite3_finalize(pStmt); + } + sqlite3_finalize(pStmt); + if( nMulti ){ runSql(p, "DELETE FROM badHash WHERE sz>1"); - nMulti = 0; writeByte(p, ORIGIN_READY); if( p->zDebugFile ) debugMessage(p, "-> ORIGIN_READY\n"); }else{ @@ -1478,11 +1487,11 @@ static void originSide(SQLiteRsync *p){ sqlite3_finalize(pInsHash); pCkHash = 0; pInsHash = 0; - if( iHash+1nPage ){ + if( mxHashnPage ){ runSql(p, "WITH RECURSIVE c(n) AS" " (VALUES(%d) UNION ALL SELECT n+1 FROM c WHERE n<%d)" " INSERT INTO badHash SELECT n, 1 FROM c", - iHash+1, p->nPage); + mxHash, p->nPage); } runSql(p, "DELETE FROM badHash WHERE pgno=%d", lockBytePage); pStmt = prepareStmt(p, @@ -1551,7 +1560,7 @@ static void sendHashMessages( "SELECT if(npg==1," " (SELECT hash(data) FROM sqlite_dbpage('replica') WHERE pgno=fpg)," " (WITH RECURSIVE c(n) AS" - " (SELECT fpg UNION ALL SELECT n+1 FROM c WHERE n REPLICA_CONFIG %u %u\n", pgno, npg); } } - writeByte(p, REPLICA_HASH); - writeBytes(p, 20, a); - if( p->zDebugFile ) debugMessage(p, "-> REPLICA_HASH %u\n", iHash); + if( a==0 ){ + if( p->zDebugFile ){ + debugMessage(p, "# Oops: No hash for %u %u\n", pgno, npg); + } + }else{ + writeByte(p, REPLICA_HASH); + writeBytes(p, 20, a); + if( p->zDebugFile ){ + debugMessage(p, "-> REPLICA_HASH %u %u (%08x...)\n", + pgno, npg, *(unsigned int*)a); + } + } p->nHashSent++; iHash = pgno + npg; nHash = npg; @@ -1770,14 +1788,15 @@ static void replicaSide(SQLiteRsync *p){ "replica is %d bytes", szOPage, szRPage); break; } - if( p->iProtocol<2 ){ + if( p->iProtocol<2 || nRPage<=100 ){ runSql(p, "WITH RECURSIVE c(n) AS" "(VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<%d)" "INSERT INTO sendHash(fpg, npg) SELECT n, 1 FROM c", nRPage); }else{ - subdivideHashRange(p, 1, nRPage); + runSql(p,"INSERT INTO sendHash VALUES(1,1)"); + subdivideHashRange(p, 2, nRPage); } sendHashMessages(p, 1, 1); runSql(p, "PRAGMA writable_schema=ON"); From 5f4d013732211afd08584e078bb02fdc0737a314 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 2 May 2025 23:50:30 +0000 Subject: [PATCH 33/42] Clean up command-line parsing. Add the undocumented -protocol option for debugging. FossilOrigin-Name: 2a52b174e6e91cd7cfca0b20b39cdd231f377c7bc073f4223574556b94b16aa5 --- manifest | 12 +++--- manifest.uuid | 2 +- tool/sqlite3_rsync.c | 87 +++++++++++++++++++++++++------------------- 3 files changed, 57 insertions(+), 44 deletions(-) diff --git a/manifest b/manifest index 3b78d8c88d..42c215830d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Now\sappears\sto\sbe\sworking.\s\sMore\stesting\sneeded.\s\sRefinement\sof\sthe\nversion-2\salgorithm\sneeded. -D 2025-05-02T22:25:40.184 +C Clean\sup\scommand-line\sparsing.\s\sAdd\sthe\sundocumented\s\n-protocol\soption\sfor\sdebugging. +D 2025-05-02T23:50:30.776 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c 7f61ac7b58fe7d0d5a6abbd1570c75bce9c160fc0d5b1b679b2213d03f047b27 +F tool/sqlite3_rsync.c 97ec284a12ccf4203ace717cb1943ac652e2723d86ef0acfca517b270516e85c F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c70330668690e7c3c55ae34137d5b2c91871432004b82b2b23a89fc3f1322a62 -R 68278fe558d911d1ca48d9ecb46d934b +P cb035181d9fb5909696b8ec8f9c3eeb7a7dfb4b50e82e1d3f2d5ad150afcc0ff +R d99c926bcfda0557c90ab8e625e5e88b U drh -Z 92a9f6cb386b9d8c6c4ee41cda50ebec +Z 1169e74b2b686e6aa9ef554d4d02a6d5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c81f190ec5..12ac9e6258 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb035181d9fb5909696b8ec8f9c3eeb7a7dfb4b50e82e1d3f2d5ad150afcc0ff +2a52b174e6e91cd7cfca0b20b39cdd231f377c7bc073f4223574556b94b16aa5 diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index 62f751ad34..cc770ea922 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -1309,7 +1309,7 @@ static void originSide(SQLiteRsync *p){ p->isReplica = 0; if( p->bCommCheck ){ infoMsg(p, "origin zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d", - p->zOrigin, p->zReplica, p->isRemote, PROTOCOL_VERSION); + p->zOrigin, p->zReplica, p->isRemote, p->iProtocol); writeByte(p, ORIGIN_END); fflush(p->pOut); }else{ @@ -1335,13 +1335,12 @@ static void originSide(SQLiteRsync *p){ if( p->nErr==0 ){ /* Send the ORIGIN_BEGIN message */ writeByte(p, ORIGIN_BEGIN); - writeByte(p, PROTOCOL_VERSION); + writeByte(p, p->iProtocol); writePow2(p, szPg); writeUint32(p, nPage); fflush(p->pOut); p->nPage = nPage; p->szPage = szPg; - p->iProtocol = PROTOCOL_VERSION; lockBytePage = (1<<30)/szPg + 1; } } @@ -1691,7 +1690,7 @@ static void replicaSide(SQLiteRsync *p){ p->isReplica = 1; if( p->bCommCheck ){ infoMsg(p, "replica zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d", - p->zOrigin, p->zReplica, p->isRemote, PROTOCOL_VERSION); + p->zOrigin, p->zReplica, p->isRemote, p->iProtocol); writeByte(p, REPLICA_END); fflush(p->pOut); } @@ -1710,9 +1709,10 @@ static void replicaSide(SQLiteRsync *p){ unsigned int nOPage = 0; unsigned int nRPage = 0, szRPage = 0; int rc = 0; + u8 iProtocol; closeDb(p); - p->iProtocol = readByte(p); + iProtocol = readByte(p); szOPage = readPow2(p); readUint32(p, &nOPage); if( p->zDebugFile ){ @@ -1720,18 +1720,19 @@ static void replicaSide(SQLiteRsync *p){ nOPage); } if( p->nErr ) break; - if( p->iProtocol>PROTOCOL_VERSION ){ + if( iProtocol>p->iProtocol ){ /* If the protocol version on the origin side is larger, send back ** a REPLICA_BEGIN message with the protocol version number of the ** replica side. This gives the origin an opportunity to resend ** a new ORIGIN_BEGIN with a reduced protocol version. */ writeByte(p, REPLICA_BEGIN); - writeByte(p, PROTOCOL_VERSION); + writeByte(p, p->iProtocol); if( p->zDebugFile ){ - debugMessage(p, "-> REPLICA_BEGIN %u\n", PROTOCOL_VERSION); + debugMessage(p, "-> REPLICA_BEGIN %u\n", p->iProtocol); } break; } + p->iProtocol = iProtocol; p->nPage = nOPage; p->szPage = szOPage; rc = sqlite3_open(":memory:", &p->db); @@ -2006,6 +2007,7 @@ int main(int argc, char const * const *argv){ #define cli_opt_val cmdline_option_value(argc, argv, ++i) memset(&ctx, 0, sizeof(ctx)); + ctx.iProtocol = PROTOCOL_VERSION; for(i=1; iPROTOCOL_VERSION ){ + ctx.iProtocol = PROTOCOL_VERSION; + } continue; } - if( strcmp(z, "-help")==0 || strcmp(z, "--help")==0 - || strcmp(z, "-?")==0 - ){ - printf("%s", zUsage); + if( strcmp(z,"-commcheck")==0 ){ /* DEBUG ONLY */ + /* Run a communication check with the remote side. Do not attempt + ** to exchange any database connection */ + ctx.bCommCheck = 1; + continue; + } + if( strcmp(z,"-arg-escape-check")==0 ){ /* DEBUG ONLY */ + /* Test the append_escaped_arg() routine by using it to render a + ** copy of the input command-line, assuming all arguments except + ** this one are filenames. */ + sqlite3_str *pStr = sqlite3_str_new(0); + int k; + for(k=0; k Date: Sat, 3 May 2025 07:00:51 +0000 Subject: [PATCH 34/42] Improvements to protocol negotiation. FossilOrigin-Name: 4f5a06e42010c3e047429f736ffb8e2e89a1eb599277c176945b57710f6713ca --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3_rsync.c | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 42c215830d..565a024081 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\scommand-line\sparsing.\s\sAdd\sthe\sundocumented\s\n-protocol\soption\sfor\sdebugging. -D 2025-05-02T23:50:30.776 +C Improvements\sto\sprotocol\snegotiation. +D 2025-05-03T07:00:51.378 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c 97ec284a12ccf4203ace717cb1943ac652e2723d86ef0acfca517b270516e85c +F tool/sqlite3_rsync.c 756ad79858feaf050a9ff60ffdb201a7a2b6a1ee620ede5743a0cef51defcdc0 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cb035181d9fb5909696b8ec8f9c3eeb7a7dfb4b50e82e1d3f2d5ad150afcc0ff -R d99c926bcfda0557c90ab8e625e5e88b +P 2a52b174e6e91cd7cfca0b20b39cdd231f377c7bc073f4223574556b94b16aa5 +R 2857adf0e10b56883ce6697a283afd6e U drh -Z 1169e74b2b686e6aa9ef554d4d02a6d5 +Z c7d74e185dc3b48152bf82febe4de310 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 12ac9e6258..f3b4164d07 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2a52b174e6e91cd7cfca0b20b39cdd231f377c7bc073f4223574556b94b16aa5 +4f5a06e42010c3e047429f736ffb8e2e89a1eb599277c176945b57710f6713ca diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index cc770ea922..760559aec2 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -1339,6 +1339,9 @@ static void originSide(SQLiteRsync *p){ writePow2(p, szPg); writeUint32(p, nPage); fflush(p->pOut); + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_BEGIN %u %u %u\n", p->iProtocol,szPg,nPage); + } p->nPage = nPage; p->szPage = szPg; lockBytePage = (1<<30)/szPg + 1; @@ -1363,6 +1366,7 @@ static void originSide(SQLiteRsync *p){ writeByte(p, p->iProtocol); writePow2(p, p->szPage); writeUint32(p, p->nPage); + fflush(p->pOut); if( p->zDebugFile ){ debugMessage(p, "-> ORIGIN_BEGIN %d %d %u\n", p->iProtocol, p->szPage, p->nPage); @@ -1694,6 +1698,7 @@ static void replicaSide(SQLiteRsync *p){ writeByte(p, REPLICA_END); fflush(p->pOut); } + if( p->iProtocol<=0 ) p->iProtocol = PROTOCOL_VERSION; /* Respond to message from the origin. The origin will initiate the ** the conversation with an ORIGIN_BEGIN message. @@ -1716,7 +1721,7 @@ static void replicaSide(SQLiteRsync *p){ szOPage = readPow2(p); readUint32(p, &nOPage); if( p->zDebugFile ){ - debugMessage(p, "<- ORIGIN_BEGIN %d %d %u\n", p->iProtocol, szOPage, + debugMessage(p, "<- ORIGIN_BEGIN %d %d %u\n", iProtocol, szOPage, nOPage); } if( p->nErr ) break; @@ -1727,6 +1732,7 @@ static void replicaSide(SQLiteRsync *p){ ** a new ORIGIN_BEGIN with a reduced protocol version. */ writeByte(p, REPLICA_BEGIN); writeByte(p, p->iProtocol); + fflush(p->pOut); if( p->zDebugFile ){ debugMessage(p, "-> REPLICA_BEGIN %u\n", p->iProtocol); } From d7b83224bb86ccbbca340bb0c39b3366cd185bcd Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 3 May 2025 10:35:32 +0000 Subject: [PATCH 35/42] Promote the --protocol option to sqlite3_rsync from being an undocumented debug option to being a supported and user-visible option. This is sometimes needed to work around bugs in prior versions running on the remote. FossilOrigin-Name: 4855e04e44e8fce2d6e37dd468eb6e9f4565c36cbc964156e65ac0449d7c212f --- manifest | 13 ++++++------- manifest.uuid | 2 +- tool/sqlite3_rsync.c | 33 ++++++++++++++++----------------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 1b3a9d2a06..634407e347 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\ssqlite3_rsync\s(in\sa\sbackwards-compatible\sway)\sso\sthat\sit\shas\sthe\nability\sto\ssend\shashes\sfor\sblocks\sof\spages\sin\saddition\sto\sindividual\spages.\nBy\sjudicious\suse\sof\sthis\scapability,\snetwork\sbandwidth\srequirement\sto\ssync\stwo\nsimilar\sdatabases\sis\sreduced. -D 2025-05-03T08:17:46.240 +C Promote\sthe\s--protocol\soption\sto\ssqlite3_rsync\sfrom\sbeing\san\sundocumented\ndebug\soption\sto\sbeing\sa\ssupported\sand\suser-visible\soption.\s\sThis\sis\s\nsometimes\sneeded\sto\swork\saround\sbugs\sin\sprior\sversions\srunning\son\sthe\nremote. +D 2025-05-03T10:35:32.727 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c 756ad79858feaf050a9ff60ffdb201a7a2b6a1ee620ede5743a0cef51defcdc0 +F tool/sqlite3_rsync.c 2b054aedddb9a8d042f865bcf7e6d056a538c6ea370075a45a1b1d5321336506 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,9 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 660a035b6ce6684d429b882133e032181cc1664f4efadf1bc0e4ae27d45071c4 4f5a06e42010c3e047429f736ffb8e2e89a1eb599277c176945b57710f6713ca -R 2857adf0e10b56883ce6697a283afd6e -T +closed 4f5a06e42010c3e047429f736ffb8e2e89a1eb599277c176945b57710f6713ca +P e5d87aaa8fe1e8c8cb63813d26851183e77809a3d36e4c16c37f88b4b4724d6d +R 274803538653b39ce860d97f8dde4625 U drh -Z c3005ffc7ac3fefab06e6e18950a9819 +Z 61278ff2cbea7da5e237a83fadf41915 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b8cdd4302e..79dd0229bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e5d87aaa8fe1e8c8cb63813d26851183e77809a3d36e4c16c37f88b4b4724d6d +4855e04e44e8fce2d6e37dd468eb6e9f4565c36cbc964156e65ac0449d7c212f diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index 760559aec2..6607e1fb06 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -30,12 +30,13 @@ static const char zUsage[] = "\n" "OPTIONS:\n" "\n" - " --exe PATH Name of the sqlite3_rsync program on the remote side\n" - " --help Show this help screen\n" - " --ssh PATH Name of the SSH program used to reach the remote side\n" - " -v Verbose. Multiple v's for increasing output\n" - " --version Show detailed version information\n" - " --wal-only Do not sync unless both databases are in WAL mode\n" + " --exe PATH Name of the sqlite3_rsync program on the remote side\n" + " --help Show this help screen\n" + " --protocol N Use sync protocol version N.\n" + " --ssh PATH Name of the SSH program used to reach the remote side\n" + " -v Verbose. Multiple v's for increasing output\n" + " --version Show detailed version information\n" + " --wal-only Do not sync unless both databases are in WAL mode\n" ; typedef unsigned char u8; @@ -2029,6 +2030,15 @@ int main(int argc, char const * const *argv){ ctx.eVerbose += numVs(z); continue; } + if( strcmp(z, "-protocol")==0 ){ + ctx.iProtocol = atoi(cli_opt_val); + if( ctx.iProtocol<1 ){ + ctx.iProtocol = 1; + }else if( ctx.iProtocol>PROTOCOL_VERSION ){ + ctx.iProtocol = PROTOCOL_VERSION; + } + continue; + } if( strcmp(z, "-ssh")==0 ){ zSsh = cli_opt_val; continue; @@ -2089,17 +2099,6 @@ int main(int argc, char const * const *argv){ zRemoteDebugFile = cli_opt_val; continue; } - if( strcmp(z, "-protocol")==0 ){ - /* DEBUG OPTION: --protocool N - ** Set the protocol version to N */ - ctx.iProtocol = atoi(cli_opt_val); - if( ctx.iProtocol<1 ){ - ctx.iProtocol = 1; - }else if( ctx.iProtocol>PROTOCOL_VERSION ){ - ctx.iProtocol = PROTOCOL_VERSION; - } - continue; - } if( strcmp(z,"-commcheck")==0 ){ /* DEBUG ONLY */ /* Run a communication check with the remote side. Do not attempt ** to exchange any database connection */ From e618674714d4ab4bbb9da3d440d3998d991ac595 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 3 May 2025 10:49:39 +0000 Subject: [PATCH 36/42] Addition summary results output when using -vvv on sqlite3_rsync. FossilOrigin-Name: c702999cfac37fdcae64d261408e58d1f49fee65434fe346db6a2a6c7f8ac54e --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3_rsync.c | 11 ++++++++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 634407e347..5156755685 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Promote\sthe\s--protocol\soption\sto\ssqlite3_rsync\sfrom\sbeing\san\sundocumented\ndebug\soption\sto\sbeing\sa\ssupported\sand\suser-visible\soption.\s\sThis\sis\s\nsometimes\sneeded\sto\swork\saround\sbugs\sin\sprior\sversions\srunning\son\sthe\nremote. -D 2025-05-03T10:35:32.727 +C Addition\ssummary\sresults\soutput\swhen\susing\s-vvv\son\ssqlite3_rsync. +D 2025-05-03T10:49:39.005 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c 2b054aedddb9a8d042f865bcf7e6d056a538c6ea370075a45a1b1d5321336506 +F tool/sqlite3_rsync.c f6a7b1577b1fceb13f3fccc7ca25a6a3b1565e22b784f4862a028d7945b48670 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e5d87aaa8fe1e8c8cb63813d26851183e77809a3d36e4c16c37f88b4b4724d6d -R 274803538653b39ce860d97f8dde4625 +P 4855e04e44e8fce2d6e37dd468eb6e9f4565c36cbc964156e65ac0449d7c212f +R d43ab97a6e993d57a1722261542d0a22 U drh -Z 61278ff2cbea7da5e237a83fadf41915 +Z bf784ea19e5f3b9d0205ad27dc713ff4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 79dd0229bf..ecfb7c1912 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4855e04e44e8fce2d6e37dd468eb6e9f4565c36cbc964156e65ac0449d7c212f +c702999cfac37fdcae64d261408e58d1f49fee65434fe346db6a2a6c7f8ac54e diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index 6607e1fb06..0a3228247b 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -40,6 +40,7 @@ static const char zUsage[] = ; typedef unsigned char u8; +typedef sqlite3_uint64 u64; /* Context for the run */ typedef struct SQLiteRsync SQLiteRsync; @@ -66,7 +67,8 @@ struct SQLiteRsync { sqlite3_uint64 nIn; /* Bytes received */ unsigned int nPage; /* Total number of pages in the database */ unsigned int szPage; /* Database page size */ - unsigned int nHashSent; /* Hashes sent (replica to origin) */ + u64 nHashSent; /* Hashes sent (replica to origin) */ + unsigned int nRound; /* Number of hash batches (replica to origin) */ unsigned int nPageSent; /* Page contents sent (origin to replica) */ }; @@ -1466,6 +1468,7 @@ static void originSide(SQLiteRsync *p){ if( p->zDebugFile ){ debugMessage(p, "<- REPLICA_READY\n"); } + p->nRound++; pStmt = prepareStmt(p,"SELECT pgno, sz FROM badHash WHERE sz>1"); if( pStmt==0 ) break; while( sqlite3_step(pStmt)==SQLITE_ROW ){ @@ -1603,6 +1606,7 @@ static void sendHashMessages( runSql(p, "DELETE FROM sendHash"); writeByte(p, REPLICA_READY); fflush(p->pOut); + p->nRound++; if( p->zDebugFile ) debugMessage(p, "-> REPLICA_READY\n", iHash); } @@ -2299,6 +2303,11 @@ int main(int argc, char const * const *argv){ printf("%s\n", zMsg); sqlite3_free(zMsg); } + if( ctx.eVerbose>=3 ){ + printf("hashes: %lld hash-rounds: %d" + " page updates: %d protocol: %d\n", + ctx.nHashSent, ctx.nRound, ctx.nPageSent, ctx.iProtocol); + } } sqlite3_free(zCmd); if( pIn!=0 && pOut!=0 ){ From 37b76bac18d42f6ebefa4cbbc9882b30f946427d Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 3 May 2025 10:55:47 +0000 Subject: [PATCH 37/42] Fix a harmless redundant variable declaration in sqlite3_rsync. FossilOrigin-Name: f8f15eff6ae50d569ed13a3b18f33eaa43453c0cb80b6007df38e880b62f45d0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3_rsync.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5156755685..debfc02291 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Addition\ssummary\sresults\soutput\swhen\susing\s-vvv\son\ssqlite3_rsync. -D 2025-05-03T10:49:39.005 +C Fix\sa\sharmless\sredundant\svariable\sdeclaration\sin\ssqlite3_rsync. +D 2025-05-03T10:55:47.615 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c f6a7b1577b1fceb13f3fccc7ca25a6a3b1565e22b784f4862a028d7945b48670 +F tool/sqlite3_rsync.c fc1f0560be26fe50252eaf2427f6f3f6f3c50e68d08a2d9cafdf3adc65e281ff F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4855e04e44e8fce2d6e37dd468eb6e9f4565c36cbc964156e65ac0449d7c212f -R d43ab97a6e993d57a1722261542d0a22 +P c702999cfac37fdcae64d261408e58d1f49fee65434fe346db6a2a6c7f8ac54e +R a9ba57abefc2ac2ac477ee0b0e67f826 U drh -Z bf784ea19e5f3b9d0205ad27dc713ff4 +Z 8fe7a400b8200d3506457658669b00b8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ecfb7c1912..dc34458e0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c702999cfac37fdcae64d261408e58d1f49fee65434fe346db6a2a6c7f8ac54e +f8f15eff6ae50d569ed13a3b18f33eaa43453c0cb80b6007df38e880b62f45d0 diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index 0a3228247b..819975f029 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -1488,7 +1488,6 @@ static void originSide(SQLiteRsync *p){ writeByte(p, ORIGIN_READY); if( p->zDebugFile ) debugMessage(p, "-> ORIGIN_READY\n"); }else{ - sqlite3_stmt *pStmt; sqlite3_finalize(pCkHash); sqlite3_finalize(pCkHashN); sqlite3_finalize(pInsHash); From d7324103b196c572a98724a5658970b4000b8c39 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 3 May 2025 15:17:21 +0000 Subject: [PATCH 38/42] Enhance sqlite3_rsync so that if the first attempt to invoke a copy of itself on the remote system using ssh fails, try again after augmenting the PATH. This enables sqlite3_rsync to work without the --exe option when the remote system is a Mac. FossilOrigin-Name: 38d4c94d8c7802101ef3bfb411002f9497fdbbbd2b4d3514cef5b76ffd66f75b --- manifest | 12 +-- manifest.uuid | 2 +- tool/sqlite3_rsync.c | 193 ++++++++++++++++++++++++++++++------------- 3 files changed, 143 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index debfc02291..37b79aec84 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\sredundant\svariable\sdeclaration\sin\ssqlite3_rsync. -D 2025-05-03T10:55:47.615 +C Enhance\ssqlite3_rsync\sso\sthat\sif\sthe\sfirst\sattempt\sto\sinvoke\sa\scopy\nof\sitself\son\sthe\sremote\ssystem\susing\sssh\sfails,\stry\sagain\safter\naugmenting\sthe\sPATH.\s\sThis\senables\ssqlite3_rsync\sto\swork\swithout\sthe\n--exe\soption\swhen\sthe\sremote\ssystem\sis\sa\sMac. +D 2025-05-03T15:17:21.853 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -2189,7 +2189,7 @@ F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c fc1f0560be26fe50252eaf2427f6f3f6f3c50e68d08a2d9cafdf3adc65e281ff +F tool/sqlite3_rsync.c dcf03373565eab543bd43926ed7e97ade744ab0599115fde1ff8aadf04fa4974 F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c702999cfac37fdcae64d261408e58d1f49fee65434fe346db6a2a6c7f8ac54e -R a9ba57abefc2ac2ac477ee0b0e67f826 +P f8f15eff6ae50d569ed13a3b18f33eaa43453c0cb80b6007df38e880b62f45d0 +R 92ec366dcda46ad4e89bcad267e1732a U drh -Z 8fe7a400b8200d3506457658669b00b8 +Z 6f2331f004388e224b4ac4ea1a14b67b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dc34458e0b..f3c2df3c41 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8f15eff6ae50d569ed13a3b18f33eaa43453c0cb80b6007df38e880b62f45d0 +38d4c94d8c7802101ef3bfb411002f9497fdbbbd2b4d3514cef5b76ffd66f75b diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index 819975f029..a6b4d61356 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -522,6 +522,53 @@ int append_escaped_arg(sqlite3_str *pStr, const char *zIn, int isFilename){ } return 0; } + +/* Add an approprate PATH= argument to the SSH command under construction +** in pStr +** +** About This Feature +** ================== +** +** On some ssh servers (Macs in particular are guilty of this) the PATH +** variable in the shell that runs the command that is sent to the remote +** host contains a limited number of read-only system directories: +** +** /usr/bin:/bin:/usr/sbin:/sbin +** +** The sqlite3_rsync executable cannot be installed into any of those +** directories because they are locked down, and so the "sqlite3_rsync" +** command cannot run. +** +** To work around this, the sqlite3_rsync command is prefixed with a PATH= +** argument, inserted by this function, to augment the PATH with additional +** directories in which the sqlite3_rsync executable can be installed. +** +** But other ssh servers are confused by this initial PATH= argument. +** Some ssh servers have a list of programs that they are allowed to run +** and will fail if the first argument is not on that list, and PATH=.... +** is not on that list. +** +** So that sqlite3_rsync can invoke itself on a remote system using ssh +** on a variety of platforms, the following algorithm is used: +** +** * First try running the sqlite3_rsync without any PATH= argument. +** If that works (and it does on a majority of systems) then we are +** done. +** +** * If the first attempt fails, then try again after adding the +** PATH= prefix argument. (This function is what adds that +** argument.) +** +** A consequence of this is that if the remote system is a Mac, the +** "ssh" command always ends up being invoked twice. If anybody knows a +** way around that problem, please bring it to the attention of the +** developers. +*/ +void add_path_argument(sqlite3_str *pStr){ + append_escaped_arg(pStr, + "PATH=$HOME/bin:/usr/local/bin:/opt/homebrew/bin:$PATH", 0); +} + /***************************************************************************** ** End of the append_escaped_arg() routine, adapted from the Fossil ** *****************************************************************************/ @@ -1968,9 +2015,9 @@ static char *hostSeparator(const char *zIn){ zIn++; } return zPath; - } + /* ** Parse command-line arguments. Dispatch subroutines to do the ** requested work. @@ -2175,74 +2222,106 @@ int main(int argc, char const * const *argv){ tmStart = currentTime(); zDiv = hostSeparator(ctx.zOrigin); if( zDiv ){ + int iRetry; if( hostSeparator(ctx.zReplica)!=0 ){ fprintf(stderr, "At least one of ORIGIN and REPLICA must be a local database\n" "You provided two remote databases.\n"); return 1; } - /* Remote ORIGIN and local REPLICA */ - sqlite3_str *pStr = sqlite3_str_new(0); - append_escaped_arg(pStr, zSsh, 1); - sqlite3_str_appendf(pStr, " -e none"); *(zDiv++) = 0; - append_escaped_arg(pStr, ctx.zOrigin, 0); - append_escaped_arg(pStr, zExe, 1); - append_escaped_arg(pStr, "--origin", 0); - if( ctx.bCommCheck ){ - append_escaped_arg(pStr, "--commcheck", 0); - if( ctx.eVerbose==0 ) ctx.eVerbose = 1; + /* Remote ORIGIN and local REPLICA */ + for(iRetry=0; 1 /*exit-via-break*/; iRetry++){ + sqlite3_str *pStr = sqlite3_str_new(0); + append_escaped_arg(pStr, zSsh, 1); + sqlite3_str_appendf(pStr, " -e none"); + append_escaped_arg(pStr, ctx.zOrigin, 0); + if( iRetry ) add_path_argument(pStr); + append_escaped_arg(pStr, zExe, 1); + append_escaped_arg(pStr, "--origin", 0); + if( ctx.bCommCheck ){ + append_escaped_arg(pStr, "--commcheck", 0); + if( ctx.eVerbose==0 ) ctx.eVerbose = 1; + } + if( zRemoteErrFile ){ + append_escaped_arg(pStr, "--errorfile", 0); + append_escaped_arg(pStr, zRemoteErrFile, 1); + } + if( zRemoteDebugFile ){ + append_escaped_arg(pStr, "--debugfile", 0); + append_escaped_arg(pStr, zRemoteDebugFile, 1); + } + if( ctx.bWalOnly ){ + append_escaped_arg(pStr, "--wal-only", 0); + } + append_escaped_arg(pStr, zDiv, 1); + append_escaped_arg(pStr, file_tail(ctx.zReplica), 1); + if( ctx.eVerbose<2 && iRetry==0 ){ + append_escaped_arg(pStr, "2>/dev/null", 0); + } + zCmd = sqlite3_str_finish(pStr); + if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); + if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ + if( iRetry>=1 ){ + fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); + return 1; + } + if( ctx.eVerbose>=2 ){ + printf("ssh FAILED. Retry with a PATH= argument...\n"); + } + continue; + } + replicaSide(&ctx); + if( ctx.nHashSent==0 && iRetry==0 ) continue; + break; } - if( zRemoteErrFile ){ - append_escaped_arg(pStr, "--errorfile", 0); - append_escaped_arg(pStr, zRemoteErrFile, 1); - } - if( zRemoteDebugFile ){ - append_escaped_arg(pStr, "--debugfile", 0); - append_escaped_arg(pStr, zRemoteDebugFile, 1); - } - if( ctx.bWalOnly ){ - append_escaped_arg(pStr, "--wal-only", 0); - } - append_escaped_arg(pStr, zDiv, 1); - append_escaped_arg(pStr, file_tail(ctx.zReplica), 1); - zCmd = sqlite3_str_finish(pStr); - if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); - if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ - fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); - return 1; - } - replicaSide(&ctx); }else if( (zDiv = hostSeparator(ctx.zReplica))!=0 ){ /* Local ORIGIN and remote REPLICA */ - sqlite3_str *pStr = sqlite3_str_new(0); - append_escaped_arg(pStr, zSsh, 1); - sqlite3_str_appendf(pStr, " -e none"); + int iRetry; *(zDiv++) = 0; - append_escaped_arg(pStr, ctx.zReplica, 0); - append_escaped_arg(pStr, zExe, 1); - append_escaped_arg(pStr, "--replica", 0); - if( ctx.bCommCheck ){ - append_escaped_arg(pStr, "--commcheck", 0); - if( ctx.eVerbose==0 ) ctx.eVerbose = 1; + for(iRetry=0; 1 /*exit-by-break*/; iRetry++){ + sqlite3_str *pStr = sqlite3_str_new(0); + append_escaped_arg(pStr, zSsh, 1); + sqlite3_str_appendf(pStr, " -e none"); + append_escaped_arg(pStr, ctx.zReplica, 0); + if( iRetry==1 ) add_path_argument(pStr); + append_escaped_arg(pStr, zExe, 1); + append_escaped_arg(pStr, "--replica", 0); + if( ctx.bCommCheck ){ + append_escaped_arg(pStr, "--commcheck", 0); + if( ctx.eVerbose==0 ) ctx.eVerbose = 1; + } + if( zRemoteErrFile ){ + append_escaped_arg(pStr, "--errorfile", 0); + append_escaped_arg(pStr, zRemoteErrFile, 1); + } + if( zRemoteDebugFile ){ + append_escaped_arg(pStr, "--debugfile", 0); + append_escaped_arg(pStr, zRemoteDebugFile, 1); + } + if( ctx.bWalOnly ){ + append_escaped_arg(pStr, "--wal-only", 0); + } + append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1); + append_escaped_arg(pStr, zDiv, 1); + if( ctx.eVerbose<2 && iRetry==0 ){ + append_escaped_arg(pStr, "2>/dev/null", 0); + } + zCmd = sqlite3_str_finish(pStr); + if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); + if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ + if( iRetry>=1 ){ + fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); + return 1; + }else if( ctx.eVerbose>=2 ){ + printf("ssh FAILED. Retry with a PATH= argument...\n"); + } + continue; + } + originSide(&ctx); + if( ctx.nHashSent==0 && iRetry==0 ) continue; + break; } - if( zRemoteErrFile ){ - append_escaped_arg(pStr, "--errorfile", 0); - append_escaped_arg(pStr, zRemoteErrFile, 1); - } - if( zRemoteDebugFile ){ - append_escaped_arg(pStr, "--debugfile", 0); - append_escaped_arg(pStr, zRemoteDebugFile, 1); - } - append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1); - append_escaped_arg(pStr, zDiv, 1); - zCmd = sqlite3_str_finish(pStr); - if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); - if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ - fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); - return 1; - } - originSide(&ctx); }else{ /* Local ORIGIN and REPLICA */ sqlite3_str *pStr = sqlite3_str_new(0); From 3b51a1ac013aa89841a7822d2864b00364f82e9d Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 6 May 2025 15:39:39 +0000 Subject: [PATCH 39/42] Fix console-I/O on Windows for DEBUG=3 builds. Broken by [925e97e6f4238f02]. FossilOrigin-Name: ccef4f7058928943be9204b2e53baaf791021e78e538396ba9f2a1d76323e8cf --- Makefile.msc | 2 +- autoconf/Makefile.msc | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 4428c4a65a..6aef671557 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -884,7 +884,7 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF !IF $(DEBUG)>2 -TCC = $(TCC) -DSQLITE_DEBUG=1 +TCC = $(TCC) -DSQLITE_DEBUG=1 -DSQLITE_USE_W32_FOR_CONSOLE_IO RCC = $(RCC) -DSQLITE_DEBUG=1 !IF $(DYNAMIC_SHELL)==0 TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index d7284af23a..dfa2dcfd5b 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -724,7 +724,7 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF !IF $(DEBUG)>2 -TCC = $(TCC) -DSQLITE_DEBUG=1 +TCC = $(TCC) -DSQLITE_DEBUG=1 -DSQLITE_USE_W32_FOR_CONSOLE_IO RCC = $(RCC) -DSQLITE_DEBUG=1 !IF $(DYNAMIC_SHELL)==0 TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE diff --git a/manifest b/manifest index 37b79aec84..3c917016ba 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Enhance\ssqlite3_rsync\sso\sthat\sif\sthe\sfirst\sattempt\sto\sinvoke\sa\scopy\nof\sitself\son\sthe\sremote\ssystem\susing\sssh\sfails,\stry\sagain\safter\naugmenting\sthe\sPATH.\s\sThis\senables\ssqlite3_rsync\sto\swork\swithout\sthe\n--exe\soption\swhen\sthe\sremote\ssystem\sis\sa\sMac. -D 2025-05-03T15:17:21.853 +C Fix\sconsole-I/O\son\sWindows\sfor\sDEBUG=3\sbuilds.\s\sBroken\sby\n[925e97e6f4238f02]. +D 2025-05-06T15:39:39.220 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d F Makefile.in c3e414df4dc8dfb12f1f6baf129fcb6d18cd0ebd3c9109370fb3fceeeef9a37a F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 -F Makefile.msc a0941cadc1f6d953204864f68860c75e149e424e246343fb9753d4261dfaf547 +F Makefile.msc 0206f28988bb6634c7e8aff05bf6cfa65d6dfe1d2b6bd95160dd99290a83dfc7 F README.md e28077cfbef795e99c9c75ed95aa7257a1166709b562076441a8506ac421b7c1 F VERSION 001dea55eb8304ec9130b6b44a32d3fc349f279d45a7e224fc0730c3cb8e2372 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 @@ -18,7 +18,7 @@ F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821 F auto.def 82c32443a91c1062f7a48beec37dbb2d8d03447b1286bce8df5ebb6d8d353f8a F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac F autoconf/Makefile.in 36516827bb5d2309422bbcbf53e873fa22ef179f5c25b1d3dc6a7255e63270b7 -F autoconf/Makefile.msc 5bc67d3912444c40c6f96d003e5c90663e51abb83d204a520110b1b2038dcd8b +F autoconf/Makefile.msc f15ad424ca2820df8e39d9157965710af0a64d87773706706a12ea4f96e3a0d8 F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f8f15eff6ae50d569ed13a3b18f33eaa43453c0cb80b6007df38e880b62f45d0 -R 92ec366dcda46ad4e89bcad267e1732a +P 38d4c94d8c7802101ef3bfb411002f9497fdbbbd2b4d3514cef5b76ffd66f75b +R 1e4612ee633cafd93cdc9b1e36fe0eea U drh -Z 6f2331f004388e224b4ac4ea1a14b67b +Z fe5bc9712ae0fa2a51da48241636dbb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f3c2df3c41..e2235b5bca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38d4c94d8c7802101ef3bfb411002f9497fdbbbd2b4d3514cef5b76ffd66f75b +ccef4f7058928943be9204b2e53baaf791021e78e538396ba9f2a1d76323e8cf From ad39fa81afe80fd5eaa283cdbc6bb91c83f25462 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 6 May 2025 16:28:44 +0000 Subject: [PATCH 40/42] Fix the sqlite3VdbeTypeofColumn() function so that it works correctly even when SQLITE_DEBUG is defined. FossilOrigin-Name: 1d5021533ed688d7a815ce75b338c72f577c14554027f88a21419935a9e68239 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3c917016ba..6f38a86bb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sconsole-I/O\son\sWindows\sfor\sDEBUG=3\sbuilds.\s\sBroken\sby\n[925e97e6f4238f02]. -D 2025-05-06T15:39:39.220 +C Fix\sthe\ssqlite3VdbeTypeofColumn()\sfunction\sso\sthat\sit\sworks\scorrectly\neven\swhen\sSQLITE_DEBUG\sis\sdefined. +D 2025-05-06T16:28:44.913 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -858,7 +858,7 @@ F src/vdbe.c 6e8030369862a64e5de35cf62a53d466ee4e9d54f4cce1219eca8914b96e956c F src/vdbe.h 31eddcffc1d14c76c2a20fe4e137e1ee43d44f370896fae14a067052801a3625 F src/vdbeInt.h 5446f60e89b2aa7cdf3ab0ec4e7b01b8732cd9d52d9092a0b8b1bf700768f784 F src/vdbeapi.c 28fab30ed0acc981aecfdcaab0a421503609078e29850eb28494816682baf0a7 -F src/vdbeaux.c 72a1c99d9300a5e0adff2c708074ac1355a619664301db474a48e147418fba05 +F src/vdbeaux.c 948c379976885a073b54cc7d8ffda087dc1a1095d1f5bb8df218796f8c933ac3 F src/vdbeblob.c b1b4032cac46b41e44b957c4d00aee9851f862dfd85ecb68116ba49884b03dfd F src/vdbemem.c e67d9c6484d868c879d20c70d00bf4a9058082f1d4058607ca15d50eb3aebc21 F src/vdbesort.c 49e366d0216c782eba287bf602384e4330d2526a22f1275492d2785ce103c79b @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 38d4c94d8c7802101ef3bfb411002f9497fdbbbd2b4d3514cef5b76ffd66f75b -R 1e4612ee633cafd93cdc9b1e36fe0eea +P ccef4f7058928943be9204b2e53baaf791021e78e538396ba9f2a1d76323e8cf +R f6da72e117d0748ab2dd41858929ab91 U drh -Z fe5bc9712ae0fa2a51da48241636dbb7 +Z 4d404117bbc4adfa4f143cc99e22f267 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e2235b5bca..784b197cc6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ccef4f7058928943be9204b2e53baaf791021e78e538396ba9f2a1d76323e8cf +1d5021533ed688d7a815ce75b338c72f577c14554027f88a21419935a9e68239 diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 6d36f7280a..a6798e62db 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1306,6 +1306,9 @@ void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ */ void sqlite3VdbeTypeofColumn(Vdbe *p, int iDest){ VdbeOp *pOp = sqlite3VdbeGetLastOp(p); +#ifdef SQLITE_DEBUG + while( pOp->opcode==OP_ReleaseReg ) pOp--; +#endif if( pOp->p3==iDest && pOp->opcode==OP_Column ){ pOp->p5 |= OPFLAG_TYPEOFARG; } From 288f35685473c82b12bd47610e473ec34524bba7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 6 May 2025 17:53:27 +0000 Subject: [PATCH 41/42] Fix a bug in the NOT NULL/IS NULL optimization of check-in [cb94350185f555c3] that can cause invalid data to be used for a column if that column has a CHECK constraint that includes the NOT NULL or IS NULL operator. Problem discovered by the [https://issues.chromium.org/issues/415397143|Chromium fuzzer]. Never seen in the wild, as far as anybody knows. FossilOrigin-Name: 2adaee9aa90f280a406007695fbc4a314806584c93d6b62b46c031492b31ec27 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 8 ++++---- src/vdbe.c | 1 + 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 6f38a86bb4..185ea27cce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3VdbeTypeofColumn()\sfunction\sso\sthat\sit\sworks\scorrectly\neven\swhen\sSQLITE_DEBUG\sis\sdefined. -D 2025-05-06T16:28:44.913 +C Fix\sa\sbug\sin\sthe\sNOT\sNULL/IS\sNULL\soptimization\sof\scheck-in\s[cb94350185f555c3]\nthat\scan\scause\sinvalid\sdata\sto\sbe\sused\sfor\sa\scolumn\sif\sthat\scolumn\shas\sa\nCHECK\sconstraint\sthat\sincludes\sthe\sNOT\sNULL\sor\sIS\sNULL\soperator.\nProblem\sdiscovered\sby\sthe\s\n[https://issues.chromium.org/issues/415397143|Chromium\sfuzzer].\s\sNever\nseen\sin\sthe\swild,\sas\sfar\sas\sanybody\sknows. +D 2025-05-06T17:53:27.367 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -736,7 +736,7 @@ F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9 F src/dbpage.c fcb1aafe00872a8aff9a7aa0ef7ff1b01e5817ec7bbd521f8f3e1e674ac8d609 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c 565d2b6403e85126d2f39f993bfbd9968c29b1100450357bc6da931469fad315 +F src/expr.c 6f184da1f36576ad1ecc48a03f14774235373c64f88d462c710834930ee6c145 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f F src/func.c 7686ea382b20e8bfe2ab9de76150c99ee7b6e83523561f3c7787e0f68cb435c2 @@ -854,7 +854,7 @@ F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c 3a20cbae9688af4c1e3754cc2520189d00762e37f60c2deb0b303360d166bba6 F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 F src/vacuum.c d580ceb395c1ae3d59da41cbfea60683ff7dd2b94ddf4d0f5657620159e2eeb7 -F src/vdbe.c 6e8030369862a64e5de35cf62a53d466ee4e9d54f4cce1219eca8914b96e956c +F src/vdbe.c 0feab5781141acca67bd5de84172fff902304274ec5cfe58609f005b8d160050 F src/vdbe.h 31eddcffc1d14c76c2a20fe4e137e1ee43d44f370896fae14a067052801a3625 F src/vdbeInt.h 5446f60e89b2aa7cdf3ab0ec4e7b01b8732cd9d52d9092a0b8b1bf700768f784 F src/vdbeapi.c 28fab30ed0acc981aecfdcaab0a421503609078e29850eb28494816682baf0a7 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ccef4f7058928943be9204b2e53baaf791021e78e538396ba9f2a1d76323e8cf -R f6da72e117d0748ab2dd41858929ab91 +P 1d5021533ed688d7a815ce75b338c72f577c14554027f88a21419935a9e68239 +R ab7250b40dfec6980bb5fa23b5ad485b U drh -Z 4d404117bbc4adfa4f143cc99e22f267 +Z 7d2b589fb6d422aa7161eac531029503 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 784b197cc6..02cea41870 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d5021533ed688d7a815ce75b338c72f577c14554027f88a21419935a9e68239 +2adaee9aa90f280a406007695fbc4a314806584c93d6b62b46c031492b31ec27 diff --git a/src/expr.c b/src/expr.c index bf15811bc4..12c94362f7 100644 --- a/src/expr.c +++ b/src/expr.c @@ -5927,11 +5927,11 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - sqlite3VdbeTypeofColumn(v, r1); + assert( regFree1==0 || regFree1==r1 ); + if( regFree1 ) sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); - testcase( regFree1==0 ); break; } case TK_BETWEEN: { @@ -6102,11 +6102,11 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_ISNULL: case TK_NOTNULL: { r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - sqlite3VdbeTypeofColumn(v, r1); + assert( regFree1==0 || regFree1==r1 ); + if( regFree1 ) sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); - testcase( regFree1==0 ); break; } case TK_BETWEEN: { diff --git a/src/vdbe.c b/src/vdbe.c index c9972fe16b..29b6f9a653 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3711,6 +3711,7 @@ case OP_MakeRecord: { zHdr += sqlite3PutVarint(zHdr, serial_type); if( pRec->n ){ assert( pRec->z!=0 ); + assert( pRec->z!=(const char*)sqlite3CtypeMap ); memcpy(zPayload, pRec->z, pRec->n); zPayload += pRec->n; } From f67058367b09ebf31dd5aaf199715ba0fd6265fe Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 May 2025 17:56:32 +0000 Subject: [PATCH 42/42] Add test cases for the NOT NULL/IS NULL optimization in CHECK constraints fix. FossilOrigin-Name: 6eb2939a6093c0796910645172d80c53055559dd57c012f1dc815d89fbf84447 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/check.test | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 185ea27cce..ffede026c3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sthe\sNOT\sNULL/IS\sNULL\soptimization\sof\scheck-in\s[cb94350185f555c3]\nthat\scan\scause\sinvalid\sdata\sto\sbe\sused\sfor\sa\scolumn\sif\sthat\scolumn\shas\sa\nCHECK\sconstraint\sthat\sincludes\sthe\sNOT\sNULL\sor\sIS\sNULL\soperator.\nProblem\sdiscovered\sby\sthe\s\n[https://issues.chromium.org/issues/415397143|Chromium\sfuzzer].\s\sNever\nseen\sin\sthe\swild,\sas\sfar\sas\sanybody\sknows. -D 2025-05-06T17:53:27.367 +C Add\stest\scases\sfor\sthe\sNOT\sNULL/IS\sNULL\soptimization\sin\sCHECK\sconstraints\sfix. +D 2025-05-06T17:56:32.330 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -1005,7 +1005,7 @@ F test/cast.test a2a3b32df86e3c0601ffa2e9f028a18796305d251801efea807092dbf374a04 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/changes.test 4377d202a487f66fc2822c1bf57c46798c8b2caf7446f4f701723b1dbb6b86f6 F test/changes2.test 07949edcc732af28cb54276bfb7d99723bccc1e905a423648bf57ac5cb0dc792 -F test/check.test 56e4ed457e9f8683b9fc56f5b964f461f6e8a8dd5a13f3d495408215d66419ed +F test/check.test 3a7972ccbaad80d496833da8714d69d9d5d4ce9e7211af1cd2a06ae488a7de12 F test/checkfault.test da6cb3d50247169efcb20bdf57863a3ccfa1d27d9e55cd324f0680096970f014 F test/chunksize.test faea11c5d6df9d392252a8dd879e1b1d68c9d3e8b7909cbed8bcec3b60c706f1 F test/cksumvfs.test 6f05dc95847c06a3dc10eee6b5ab1351d78314a52d0db15717c9388f4cb96646 @@ -2207,8 +2207,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1d5021533ed688d7a815ce75b338c72f577c14554027f88a21419935a9e68239 -R ab7250b40dfec6980bb5fa23b5ad485b -U drh -Z 7d2b589fb6d422aa7161eac531029503 +P 2adaee9aa90f280a406007695fbc4a314806584c93d6b62b46c031492b31ec27 +R 785918686456a4e780990cb7c7171f36 +U dan +Z bab07d1fb4803bf298a1c8473b03d461 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 02cea41870..dc7c0bb9b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2adaee9aa90f280a406007695fbc4a314806584c93d6b62b46c031492b31ec27 +6eb2939a6093c0796910645172d80c53055559dd57c012f1dc815d89fbf84447 diff --git a/test/check.test b/test/check.test index 10d1cf4be6..c3beb2f5d8 100644 --- a/test/check.test +++ b/test/check.test @@ -612,4 +612,36 @@ do_catchsql_test 12.81 { INSERT INTO t1(a) VALUES(456); } {1 {CHECK constraint failed: a NOT BETWEEN +a AND 999999}} +#------------------------------------------------------------------------- + +reset_db + +do_execsql_test 13.1.0 { + CREATE TABLE Table0 (Col0 , CHECK(Table0.Col0 NOT NULL ) ) ; + REPLACE INTO Table0 VALUES (hex(randomblob(100000))); +} +integrity_check 13.1.1 +do_execsql_test 13.1.2 { + UPDATE OR REPLACE Table0 SET Col0 = Table0.Col0 ; +} +integrity_check 13.1.3 +do_execsql_test 13.1.4 { + SELECT length(col0) FROM table0; +} {200000} + +do_execsql_test 13.2.0 { + CREATE TABLE t2 (x , CHECK((NOT (x ISNULL) ))); + REPLACE INTO t2 VALUES (hex(randomblob(100000))); +} +do_execsql_test 13.2.1 { + SELECT length(x) FROM t2 +} {200000} +do_execsql_test 13.2.2 { + UPDATE OR REPLACE t2 SET x = x; +} +do_execsql_test 13.2.3 { + SELECT length(x) FROM t2 +} {200000} +integrity_check 13.2.4 + finish_test