From 5b6b7034615f009923fe46ee6f1fcebf23c66fdd Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 7 Dec 2023 12:55:39 +0000 Subject: [PATCH 01/80] Improved detection of corrupt JSONB in the jsonReturnFromBlob() function. FossilOrigin-Name: b014736c1f80ccc46fb4b24ac04310a6ce5cb5b6653665efff366cb3bc742257 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 43 +++++++++++++++++++++++++++++++------------ 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 10b2d5cbfd..2175694e92 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarning\sabout\sshadowed\svariable\sin\sfts5_index.c. -D 2023-12-06T18:34:59.649 +C Improved\sdetection\sof\scorrupt\sJSONB\sin\sthe\sjsonReturnFromBlob()\sfunction. +D 2023-12-07T12:55:39.473 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c c2e0fea06f40fb0319ed132fc181a25623585c943e08c690b522f216886ba316 +F src/json.c 674cabf943f30d735c8ffd50e4ae1b4e3ccb52fd7cddae4c556ebcf7f410f0c8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec0ae4030968c782af48d1c776351c14b2ada21d40aeb97915f33df30706e18f -R edfe66d6aa99b5210c61e8b6b973888d -U dan -Z 66c944797bd4988ebf2844c99b4718ad +P ee70e4c1c9c41617850228e48d8df44f105cf2fbbe789340ceca6f27ad6ce5eb +R 00cfbe2052391ca2d7a17dea6eb84d8b +U drh +Z f70f3172c97aeaef0feb684dd3865d25 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 414f169a76..0629939106 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee70e4c1c9c41617850228e48d8df44f105cf2fbbe789340ceca6f27ad6ce5eb \ No newline at end of file +b014736c1f80ccc46fb4b24ac04310a6ce5cb5b6653665efff366cb3bc742257 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 46d7a3fa8c..850e2241e6 100644 --- a/src/json.c +++ b/src/json.c @@ -2618,14 +2618,17 @@ static void jsonReturnFromBlob( } switch( pParse->aBlob[i] & 0x0f ){ case JSONB_NULL: { + if( sz ) goto returnfromblob_malformed; sqlite3_result_null(pCtx); break; } case JSONB_TRUE: { + if( sz ) goto returnfromblob_malformed; sqlite3_result_int(pCtx, 1); break; } case JSONB_FALSE: { + if( sz ) goto returnfromblob_malformed; sqlite3_result_int(pCtx, 0); break; } @@ -2634,16 +2637,25 @@ static void jsonReturnFromBlob( sqlite3_int64 iRes = 0; char *z; int bNeg = 0; - char x = (char)pParse->aBlob[i+n]; - if( x=='-' && ALWAYS(sz>0) ){ n++; sz--; bNeg = 1; } + char x; + if( sz==0 ) goto returnfromblob_malformed; + x = (char)pParse->aBlob[i+n]; + if( x=='-' ){ + if( sz<2 ) goto returnfromblob_malformed; + n++; + sz--; + bNeg = 1; + } z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); - if( z==0 ) return; + if( z==0 ) goto returnfromblob_oom; rc = sqlite3DecOrHexToI64(z, &iRes); sqlite3DbFree(db, z); - if( rc<=1 ){ + if( rc==0 ){ sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); }else if( rc==3 && bNeg ){ sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else if( rc==1 ){ + goto returnfromblob_malformed; }else{ if( bNeg ){ n--; sz++; } goto to_double; @@ -2654,11 +2666,13 @@ static void jsonReturnFromBlob( case JSONB_FLOAT: { double r; char *z; + if( sz==0 ) goto returnfromblob_malformed; to_double: z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); - if( z==0 ) return; - sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + if( z==0 ) goto returnfromblob_oom; + rc = sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); sqlite3DbFree(db, z); + if( rc<=0 ) goto returnfromblob_malformed; sqlite3_result_double(pCtx, r); break; } @@ -2677,10 +2691,7 @@ static void jsonReturnFromBlob( u32 nOut = sz; z = (const char*)&pParse->aBlob[i+n]; zOut = sqlite3_malloc( nOut+1 ); - if( zOut==0 ){ - sqlite3_result_error_nomem(pCtx); - break; - } + if( zOut==0 ) goto returnfromblob_oom; for(iIn=iOut=0; iIn Date: Thu, 7 Dec 2023 13:14:34 +0000 Subject: [PATCH 02/80] Add ALWAYS() on branches added in [ec0ae4030968c782] that are always true. FossilOrigin-Name: 451cef8609e96dd9244818adc5c6f240544694bcb4ae620e88f90e403e59d70f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 4 ++-- src/select.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 2175694e92..89fc123d14 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sdetection\sof\scorrupt\sJSONB\sin\sthe\sjsonReturnFromBlob()\sfunction. -D 2023-12-07T12:55:39.473 +C Add\sALWAYS()\son\sbranches\sadded\sin\s[ec0ae4030968c782]\sthat\sare\salways\strue. +D 2023-12-07T13:14:34.585 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -686,7 +686,7 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c f4dbcca060fa95539f6705b5ec4cb5eeaba5c0d84ff64efca8edab7a1d776807 +F src/expr.c 05278def9c186b5875d6903ea26148c7461b9ce0344f0fd7be9a0dfea0a4538a F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 @@ -736,7 +736,7 @@ F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c a5058ae54211dc49faade7b78f21c69a3fa3ee652eb270fba6881e28e30e0497 +F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445 F src/shell.c.in 9b6c3e641de45651ad0b5e9c26cd2f72efabee28179a5315d15c54239515ee3a F src/sqlite.h.in d93a4821d2f792467a60f7dc81268d1bb8634f40c31694ef254cab4f9921f96a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ee70e4c1c9c41617850228e48d8df44f105cf2fbbe789340ceca6f27ad6ce5eb -R 00cfbe2052391ca2d7a17dea6eb84d8b +P b014736c1f80ccc46fb4b24ac04310a6ce5cb5b6653665efff366cb3bc742257 +R 4ecf43441052e986caaa2b649977f529 U drh -Z f70f3172c97aeaef0feb684dd3865d25 +Z 9ea0563bc67b852b5f4309987080e02f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0629939106..12ad7048bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b014736c1f80ccc46fb4b24ac04310a6ce5cb5b6653665efff366cb3bc742257 \ No newline at end of file +451cef8609e96dd9244818adc5c6f240544694bcb4ae620e88f90e403e59d70f \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 047d2b6b39..b5e903d456 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1405,7 +1405,7 @@ void sqlite3ExprDelete(sqlite3 *db, Expr *p){ if( p ) sqlite3ExprDeleteNN(db, p); } void sqlite3ExprDeleteGeneric(sqlite3 *db, void *p){ - if( p ) sqlite3ExprDeleteNN(db, (Expr*)p); + if( ALWAYS(p) ) sqlite3ExprDeleteNN(db, (Expr*)p); } /* @@ -2239,7 +2239,7 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ if( pList ) exprListDeleteNN(db, pList); } void sqlite3ExprListDeleteGeneric(sqlite3 *db, void *pList){ - if( pList ) exprListDeleteNN(db, (ExprList*)pList); + if( ALWAYS(pList) ) exprListDeleteNN(db, (ExprList*)pList); } /* diff --git a/src/select.c b/src/select.c index f61b34f5fe..1beaf5307d 100644 --- a/src/select.c +++ b/src/select.c @@ -185,7 +185,7 @@ void sqlite3SelectDelete(sqlite3 *db, Select *p){ if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1); } void sqlite3SelectDeleteGeneric(sqlite3 *db, void *p){ - if( p ) clearSelect(db, (Select*)p, 1); + if( ALWAYS(p) ) clearSelect(db, (Select*)p, 1); } /* From 5a238ffcae3e670424797224296c651c2457f922 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 7 Dec 2023 14:09:25 +0000 Subject: [PATCH 03/80] Rework the jsonEachPathLength() routine in json_tree() so that it is less susceptible to problems due to goofy object labels. FossilOrigin-Name: 858b76a00e8ff55215f7a2e6a4cd77fc4d4f98dea7224cd90488744f5ce246a4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 31 ++++++++++++++----------------- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 89fc123d14..de24979e9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sALWAYS()\son\sbranches\sadded\sin\s[ec0ae4030968c782]\sthat\sare\salways\strue. -D 2023-12-07T13:14:34.585 +C Rework\sthe\sjsonEachPathLength()\sroutine\sin\sjson_tree()\sso\sthat\sit\sis\nless\ssusceptible\sto\sproblems\sdue\sto\sgoofy\sobject\slabels. +D 2023-12-07T14:09:25.017 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 674cabf943f30d735c8ffd50e4ae1b4e3ccb52fd7cddae4c556ebcf7f410f0c8 +F src/json.c 3d64e42894afcda2efe8b112e8e2accd66fbeb150d7b1a073f20fc4a232faa37 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b014736c1f80ccc46fb4b24ac04310a6ce5cb5b6653665efff366cb3bc742257 -R 4ecf43441052e986caaa2b649977f529 +P 451cef8609e96dd9244818adc5c6f240544694bcb4ae620e88f90e403e59d70f +R 7ecf4ae92b8b647771c0ed38f3fc271f U drh -Z 9ea0563bc67b852b5f4309987080e02f +Z 2abe6b160a42e7f20184ba1ba8e11b5c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 12ad7048bf..edeb1479b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -451cef8609e96dd9244818adc5c6f240544694bcb4ae620e88f90e403e59d70f \ No newline at end of file +858b76a00e8ff55215f7a2e6a4cd77fc4d4f98dea7224cd90488744f5ce246a4 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 850e2241e6..6a4a0babfc 100644 --- a/src/json.c +++ b/src/json.c @@ -4521,23 +4521,20 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ */ static int jsonEachPathLength(JsonEachCursor *p){ u32 n = p->path.nUsed; - const char *z = p->path.zBuf; - if( p->iRowid==0 && p->bRecursive && n>1 ){ - if( z[n-1]==']' ){ - do{ - assert( n>1 ); - n--; - }while( z[n]!='[' ); - }else if( z[n-1]=='"' ){ - do{ - assert( n>1 ); - n--; - }while( z[n]!='.' || z[n+1]!='"' ); - }else{ - do{ - assert( n>1 ); - n--; - }while( z[n]!='.' ); + char *z = p->path.zBuf; + if( p->iRowid==0 && p->bRecursive && n>=2 ){ + while( n>1 ){ + n--; + if( z[n]=='[' || z[n]=='.' ){ + u32 x, sz = 0; + char cSaved = z[n]; + z[n] = 0; + assert( p->sParse.eEdit==0 ); + x = jsonLookupStep(&p->sParse, 0, z+1, 0); + z[n] = cSaved; + if( JSON_LOOKUP_ISERROR(x) ) continue; + if( x + jsonbPayloadSize(&p->sParse, x, &sz) == p->i ) break; + } } } return n; From 2f3ccb8886b8bfd700e7b55f9adb46eb8f48401e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 7 Dec 2023 14:41:58 +0000 Subject: [PATCH 04/80] Different fix for the fts5 COMMIT-following-OOM problem first fixed by [fba3129d]. This one does not cause problems if an fts5 table is renamed and then dropped within the same transaction. FossilOrigin-Name: d8c6b246944934a7a6e027b3f5b986fd64a19dd5c5c5175f4ea8586da59a6764 --- ext/fts5/fts5_index.c | 14 ++++++++++++-- ext/fts5/fts5_main.c | 30 +++++++----------------------- ext/fts5/test/fts5misc.test | 16 +++++++++++++++- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 5 files changed, 44 insertions(+), 36 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 7ec1976d33..1910290e8c 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -360,6 +360,7 @@ struct Fts5Index { /* Error state. */ int rc; /* Current error code */ + int flushRc; /* State used by the fts5DataXXX() functions. */ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ @@ -4120,6 +4121,7 @@ static void fts5IndexDiscardData(Fts5Index *p){ sqlite3Fts5HashClear(p->pHash); p->nPendingData = 0; p->nPendingRow = 0; + p->flushRc = SQLITE_OK; } p->nContentlessDelete = 0; } @@ -5701,6 +5703,10 @@ static void fts5FlushOneHash(Fts5Index *p){ */ static void fts5IndexFlush(Fts5Index *p){ /* Unless it is empty, flush the hash table to disk */ + if( p->flushRc ){ + p->rc = p->flushRc; + return; + } if( p->nPendingData || p->nContentlessDelete ){ assert( p->pHash ); fts5FlushOneHash(p); @@ -5709,6 +5715,8 @@ static void fts5IndexFlush(Fts5Index *p){ p->nPendingData = 0; p->nPendingRow = 0; p->nContentlessDelete = 0; + }else if( p->nPendingData || p->nContentlessDelete ){ + p->flushRc = p->rc; } } } @@ -6888,14 +6896,16 @@ static Fts5Iter *fts5SetupTokendataIter( int iLvl, iSeg, ii; pNew = fts5MultiIterAlloc(p, pStruct->nSegment); - if( pNew==0 ) break; - if( pSmall ){ fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p); fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0"); }else{ fts5BufferSet(&p->rc, &bSeek, nToken, pToken); } + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + break; + } pNewIter = &pNew->aSeg[0]; pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index d35e998da0..6d3d147f21 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -118,7 +118,7 @@ struct Fts5FullTable { Fts5Global *pGlobal; /* Global (connection wide) data */ Fts5Cursor *pSortCsr; /* Sort data from this cursor */ int iSavepoint; /* Successful xSavepoint()+1 */ - int bInSavepoint; + #ifdef SQLITE_DEBUG struct Fts5TransactionState ts; #endif @@ -2667,9 +2667,7 @@ static int fts5RenameMethod( ){ int rc; Fts5FullTable *pTab = (Fts5FullTable*)pVtab; - pTab->bInSavepoint = 1; rc = sqlite3Fts5StorageRename(pTab->pStorage, zName); - pTab->bInSavepoint = 0; return rc; } @@ -2686,26 +2684,12 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; int rc = SQLITE_OK; - char *zSql = 0; + fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); - - if( pTab->bInSavepoint==0 ){ - zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')", - pTab->p.pConfig->zDb, pTab->p.pConfig->zName, pTab->p.pConfig->zName - ); - if( zSql ){ - pTab->bInSavepoint = 1; - rc = sqlite3_exec(pTab->p.pConfig->db, zSql, 0, 0, 0); - pTab->bInSavepoint = 0; - sqlite3_free(zSql); - }else{ - rc = SQLITE_NOMEM; - } - if( rc==SQLITE_OK ){ - pTab->iSavepoint = iSavepoint+1; - } + rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; } - return rc; } @@ -2966,7 +2950,7 @@ static int fts5ShadowName(const char *zName){ ** if anything is found amiss. Return a NULL pointer if everything is ** OK. */ -static int fts5Integrity( +static int fts5IntegrityMethod( sqlite3_vtab *pVtab, /* the FTS5 virtual table to check */ const char *zSchema, /* Name of schema in which this table lives */ const char *zTabname, /* Name of the table itself */ @@ -3024,7 +3008,7 @@ static int fts5Init(sqlite3 *db){ /* xRelease */ fts5ReleaseMethod, /* xRollbackTo */ fts5RollbackToMethod, /* xShadowName */ fts5ShadowName, - /* xIntegrity */ fts5Integrity + /* xIntegrity */ fts5IntegrityMethod }; int rc; diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index d67d79e29f..221f3996a7 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -91,7 +91,6 @@ do_execsql_test 2.2.1 { INSERT INTO vt0(c0) VALUES ('xyz'); } -breakpoint do_execsql_test 2.2.2 { ALTER TABLE t0 RENAME TO t1; } @@ -500,6 +499,21 @@ do_execsql_test 17.5 { SELECT c0 FROM t0 WHERE c0 GLOB '*faul*'; } {assertionfaultproblem} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 18.0 { + BEGIN; + CREATE VIRTUAL TABLE t1 USING fts5(text); + ALTER TABLE t1 RENAME TO t2; +} + +do_execsql_test 18.1 { + DROP TABLE t2; +} + +do_execsql_test 18.2 { + COMMIT; +} finish_test diff --git a/manifest b/manifest index de24979e9b..e2fd3a129b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rework\sthe\sjsonEachPathLength()\sroutine\sin\sjson_tree()\sso\sthat\sit\sis\nless\ssusceptible\sto\sproblems\sdue\sto\sgoofy\sobject\slabels. -D 2023-12-07T14:09:25.017 +C Different\sfix\sfor\sthe\sfts5\sCOMMIT-following-OOM\sproblem\sfirst\sfixed\sby\s[fba3129d].\sThis\sone\sdoes\snot\scause\sproblems\sif\san\sfts5\stable\sis\srenamed\sand\sthen\sdropped\swithin\sthe\ssame\stransaction. +D 2023-12-07T14:41:58.377 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,8 +96,8 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 111fdb22e4d78a5c3813ac2c782409b3567291a48f19ae834cd50268c8e9fafc -F ext/fts5/fts5_main.c fb7ec495d663f40d18e420e1986316591041a70e1e4b4696ab2a7384e4c7fd7a +F ext/fts5/fts5_index.c 6d3571d6bcecf7bc2c09885e4ce563c9aeaec8e22551cddb025afd85a6b6130c +F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -183,7 +183,7 @@ F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fc F ext/fts5/test/fts5matchinfo.test 10c9a6f7fe61fb132299c4183c012770b10c4d5c2f2edb6df0b6607f683d737a F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test 5ca82f2a5ee016b0842043155d1382f98a34d0d86b2791165a44d7807f6e0f54 +F ext/fts5/test/fts5misc.test dd97c86c9cbc3e587067e640f6ce88842cfbf5d23bb0e0fbb7f6707623b2d505 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 451cef8609e96dd9244818adc5c6f240544694bcb4ae620e88f90e403e59d70f -R 7ecf4ae92b8b647771c0ed38f3fc271f -U drh -Z 2abe6b160a42e7f20184ba1ba8e11b5c +P 858b76a00e8ff55215f7a2e6a4cd77fc4d4f98dea7224cd90488744f5ce246a4 +R 80da091f0378004e59c1289240cd4ed3 +U dan +Z f4bdf66e363ead249a9b82f208481974 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index edeb1479b6..f1f52cb776 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -858b76a00e8ff55215f7a2e6a4cd77fc4d4f98dea7224cd90488744f5ce246a4 \ No newline at end of file +d8c6b246944934a7a6e027b3f5b986fd64a19dd5c5c5175f4ea8586da59a6764 \ No newline at end of file From 5f2f8d002861e5753ad2366bb5b583c67ebbf303 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 7 Dec 2023 18:41:49 +0000 Subject: [PATCH 05/80] Fix a problem with handling OOM and other errors in fts5 when querying tokendata=1 tables. FossilOrigin-Name: bc911ab5953532956510c199be72b1d3c556f2d0ddbd7fc0ae6f5f917b337b48 --- ext/fts5/fts5_index.c | 2 +- ext/fts5/test/fts5corrupt5.test | 158 ++++++++++++++++++++++++++++++++ manifest | 14 +-- manifest.uuid | 2 +- 4 files changed, 167 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 1910290e8c..a9e28df1d7 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6887,7 +6887,7 @@ static Fts5Iter *fts5SetupTokendataIter( fts5IndexFlush(p); pStruct = fts5StructureRead(p); - while( 1 ){ + while( p->rc==SQLITE_OK ){ Fts5Iter *pPrev = pSet ? pSet->apIter[pSet->nIter-1] : 0; Fts5Iter *pNew = 0; Fts5SegIter *pNewIter = 0; diff --git a/ext/fts5/test/fts5corrupt5.test b/ext/fts5/test/fts5corrupt5.test index 60910dce7f..fa0c314c05 100644 --- a/ext/fts5/test/fts5corrupt5.test +++ b/ext/fts5/test/fts5corrupt5.test @@ -966,6 +966,164 @@ do_catchsql_test 6.2 { UPDATE t1 SET content=randomblob(500) WHERE t1; } {1 {constraint failed}} +#------------------------------------------------------------------------- +reset_db +do_test 7.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 40960 pagesize 4096 filename crash-d8b4a99207c10b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 62 00 0f 97 0f 40 ..........b....@ +| 112: 0e d5 0e 75 0e 18 0d c0 0d 66 0d 0f 0c a4 0c 44 ...u.....f.....D +| 128: 0b ec 0b a7 0b 62 00 00 00 00 00 00 00 00 00 00 .....b.......... +| 2912: 00 00 43 0d 06 17 11 11 08 75 74 61 62 6c 65 74 ..C......utablet +| 2928: 34 74 34 43 52 45 41 54 45 20 56 49 52 54 55 41 4t4CREATE VIRTUA +| 2944: 4c 20 54 41 42 4c 45 20 74 34 20 55 53 49 4e 47 L TABLE t4 USING +| 2960: 20 66 74 73 35 76 6f 63 61 62 28 27 74 32 27 2c fts5vocab('t2', +| 2976: 20 27 72 6f 77 27 29 43 0c 06 17 11 11 08 75 74 'row')C......ut +| 2992: 61 62 6c 65 74 33 74 33 43 52 45 41 54 45 20 56 ablet3t3CREATE V +| 3008: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 33 20 IRTUAL TABLE t3 +| 3024: 55 53 49 4e 47 20 66 74 73 35 76 6f 63 61 62 28 USING fts5vocab( +| 3040: 27 74 31 27 2c 20 27 72 6f 77 27 29 56 0b 06 17 't1', 'row')V... +| 3056: 1f 1f 01 7d 74 61 62 6c 65 74 32 5f 63 6f 6e 66 ....tablet2_conf +| 3072: 69 67 74 32 5f 63 6f 6e 66 69 67 0a 43 52 45 41 igt2_config.CREA +| 3088: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 63 6f 6e TE TABLE 't2_con +| 3104: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3120: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3136: 4f 57 49 44 5e 0a 07 17 21 21 01 81 07 74 61 62 OWID^...!!...tab +| 3152: 6c 65 74 32 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 let2_contentt2_c +| 3168: 6f 6e 74 65 6e 74 09 43 52 45 41 54 45 20 54 41 ontent.CREATE TA +| 3184: 42 4c 45 20 27 74 32 5f 63 6f 6e 74 65 6e 74 27 BLE 't2_content' +| 3200: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3216: 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c ARY KEY, c0, c1, +| 3232: 20 63 32 29 69 09 07 17 19 19 01 81 2d 74 61 62 c2)i.......-tab +| 3248: 6c 65 74 32 5f 69 64 78 74 32 5f 69 64 78 08 43 let2_idxt2_idx.C +| 3264: 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 32 5f REATE TABLE 't2_ +| 3280: 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d idx'(segid, term +| 3296: 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 , pgno, PRIMARY +| 3312: 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 KEY(segid, term) +| 3328: 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 ) WITHOUT ROWIDU +| 3344: 08 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 32 5f ........tablet2_ +| 3360: 64 61 74 61 74 32 5f 64 61 74 61 07 43 52 45 41 datat2_data.CREA +| 3376: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 64 61 74 TE TABLE 't2_dat +| 3392: 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 a'(id INTEGER PR +| 3408: 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b IMARY KEY, block +| 3424: 20 42 4c 4f 42 29 58 07 07 17 11 11 08 81 1d 74 BLOB)X........t +| 3440: 61 62 6c 65 74 32 74 32 43 52 45 41 54 45 20 56 ablet2t2CREATE V +| 3456: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 32 20 IRTUAL TABLE t2 +| 3472: 55 53 49 4e 47 20 66 74 73 35 28 27 61 27 2c 5b USING fts5('a',[ +| 3488: 62 5d 2c 22 63 22 2c 64 65 74 61 69 6c 3d 6e 6f b],.c.,detail=no +| 3504: 6e 65 2c 63 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 ne,columnsize=0) +| 3520: 56 06 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f V.......tablet1_ +| 3536: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 06 configt1_config. +| 3552: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3568: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41 _config'(k PRIMA +| 3584: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f RY KEY, v) WITHO +| 3600: 55 54 20 52 4f 57 49 44 5b 05 07 17 21 21 01 81 UT ROWID[...!!.. +| 3616: 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 .tablet1_docsize +| 3632: 74 31 5f 64 6f 63 73 69 7a 65 05 43 52 45 41 54 t1_docsize.CREAT +| 3648: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs +| 3664: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20 ize'(id INTEGER +| 3680: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20 PRIMARY KEY, sz +| 3696: 42 4c 4f 42 29 5e 04 07 17 21 21 01 81 07 74 61 BLOB)^...!!...ta +| 3712: 62 6c 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f blet1_contentt1_ +| 3728: 63 6f 6e 74 65 6e 74 04 43 52 45 41 54 45 20 54 content.CREATE T +| 3744: 41 42 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74 ABLE 't1_content +| 3760: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 '(id INTEGER PRI +| 3776: 4d 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 MARY KEY, c0, c1 +| 3792: 2c 20 63 32 29 69 03 07 17 19 19 01 81 2d 74 61 , c2)i.......-ta +| 3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03 blet1_idxt1_idx. +| 3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 _idx'(segid, ter +| 3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 m, pgno, PRIMARY +| 3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d KEY(segid, term +| 3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 )) WITHOUT ROWID +| 3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 U........tablet1 +| 3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45 _datat1_data.CRE +| 3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 ATE TABLE 't1_da +| 3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 ta'(id INTEGER P +| 3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 RIMARY KEY, bloc +| 3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b k BLOB)g.......; +| 4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE +| 4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 62 20 USING fts5(a,b +| 4048: 75 6e 69 6e 64 65 78 65 64 2c 63 2c 74 6f 6b 65 unindexed,c,toke +| 4064: 6e 69 7a 65 3d 22 70 6f 72 74 65 72 20 61 73 63 nize=.porter asc +| 4080: 69 69 22 2c 74 6f 6b 65 6e 64 61 74 61 3d 31 29 ii.,tokendata=1) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 7f f1 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-d8b4a99207c10b.db +}]} {} + +do_catchsql_test 7.1 { + SELECT snippet(t1, -1, '.', '..', '[', ']'), + highlight(t1, 2, '[', ']') + FROM t1('g + h') + WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank; +} {1 {database disk image is malformed}} + + sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/manifest b/manifest index e2fd3a129b..d37c195d80 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Different\sfix\sfor\sthe\sfts5\sCOMMIT-following-OOM\sproblem\sfirst\sfixed\sby\s[fba3129d].\sThis\sone\sdoes\snot\scause\sproblems\sif\san\sfts5\stable\sis\srenamed\sand\sthen\sdropped\swithin\sthe\ssame\stransaction. -D 2023-12-07T14:41:58.377 +C Fix\sa\sproblem\swith\shandling\sOOM\sand\sother\serrors\sin\sfts5\swhen\squerying\stokendata=1\stables. +D 2023-12-07T18:41:49.112 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 6d3571d6bcecf7bc2c09885e4ce563c9aeaec8e22551cddb025afd85a6b6130c +F ext/fts5/fts5_index.c 7a459f96b283477063f83f40f687859c1264aff42769829ea1ef665f9b7342b8 F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -145,7 +145,7 @@ F ext/fts5/test/fts5corrupt.test b6d4034b682bb3387bc44c510c71b3c67d4349e4df13949 F ext/fts5/test/fts5corrupt2.test 99e7e23a58b4d89eb7167c6de1669cbc595cd3c79ab333e0eb56405473319e77 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78 F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3 -F ext/fts5/test/fts5corrupt5.test 38a238df26c4de471e1c4b98f8de6c902bc692577a1c08d0ff4f2251f3559644 +F ext/fts5/test/fts5corrupt5.test 4f1b96f740a50faa0db1e7a9ff72ea20db5b83564cacc25c7ee23560f2bcb0c2 F ext/fts5/test/fts5corrupt6.test bf8eeae07825b088b9665d9d8e4accbd8dc9bf3cb85b6c64cf6c9e18ccc420a4 F ext/fts5/test/fts5corrupt7.test 80ad7f683a8bda2404731bb77e8c3dbbb620c1f6cc583cca8239f6accd6338c0 F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 858b76a00e8ff55215f7a2e6a4cd77fc4d4f98dea7224cd90488744f5ce246a4 -R 80da091f0378004e59c1289240cd4ed3 +P d8c6b246944934a7a6e027b3f5b986fd64a19dd5c5c5175f4ea8586da59a6764 +R cf423a1f60ce55f260a5bafd492cd4f3 U dan -Z f4bdf66e363ead249a9b82f208481974 +Z 7283dc8d98cf6fe28540e176c5956186 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f1f52cb776..9849120d16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8c6b246944934a7a6e027b3f5b986fd64a19dd5c5c5175f4ea8586da59a6764 \ No newline at end of file +bc911ab5953532956510c199be72b1d3c556f2d0ddbd7fc0ae6f5f917b337b48 \ No newline at end of file From 1ddcf7dd95968a470c37437fac8a72cb215a2167 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 7 Dec 2023 19:08:25 +0000 Subject: [PATCH 06/80] Fix a null-pointer dereference in fts5 tokendata=1 code. FossilOrigin-Name: d69fa8f0504887f968d9a190ecb889ddb40bb1b56d0d4479f9819c106aec719b --- ext/fts5/fts5_index.c | 2 +- ext/fts5/test/fts5corrupt5.test | 98 +++++++++++++++++++++++++++++++++ manifest | 14 ++--- manifest.uuid | 2 +- 4 files changed, 107 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a9e28df1d7..735d06c29f 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -7221,7 +7221,7 @@ int sqlite3Fts5IterToken( */ void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - if( pIter->pTokenDataIter ){ + if( pIter && pIter->pTokenDataIter ){ pIter->pTokenDataIter->nMap = 0; } } diff --git a/ext/fts5/test/fts5corrupt5.test b/ext/fts5/test/fts5corrupt5.test index fa0c314c05..a56467376b 100644 --- a/ext/fts5/test/fts5corrupt5.test +++ b/ext/fts5/test/fts5corrupt5.test @@ -1123,6 +1123,104 @@ do_catchsql_test 7.1 { WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank; } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +reset_db +do_test 8.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 20480 pagesize 4096 filename crash-d57c01958e48ab.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 05 0e 10 00 0f 97 0f 40 ...............@ +| 112: 0e d5 0e 68 0e 10 01 00 00 00 00 00 00 00 00 00 ...h............ +| 3600: 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f V.......tablet1_ +| 3616: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 05 configt1_config. +| 3632: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3648: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41 _config'(k PRIMA +| 3664: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f RY KEY, v) WITHO +| 3680: 55 54 20 52 4f 57 49 44 6b 04 07 17 21 21 01 81 UT ROWIDk...!!.. +| 3696: 21 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 !tablet1_docsize +| 3712: 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 45 41 54 t1_docsize.CREAT +| 3728: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs +| 3744: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20 ize'(id INTEGER +| 3760: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20 PRIMARY KEY, sz +| 3776: 42 4c 4f 42 2c 20 6f 72 69 67 69 6e 20 49 4e 54 BLOB, origin INT +| 3792: 45 47 45 52 29 69 03 07 17 19 19 01 81 2d 74 61 EGER)i.......-ta +| 3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03 blet1_idxt1_idx. +| 3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 _idx'(segid, ter +| 3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 m, pgno, PRIMARY +| 3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d KEY(segid, term +| 3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 )) WITHOUT ROWID +| 3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 U........tablet1 +| 3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45 _datat1_data.CRE +| 3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 ATE TABLE 't1_da +| 3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 ta'(id INTEGER P +| 3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 RIMARY KEY, bloc +| 3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b k BLOB)g.......; +| 4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE +| 4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62 USING fts5(a, b +| 4048: 2c 20 63 6f 6e 74 65 6e 74 3d 27 27 2c 20 63 6f , content='', co +| 4064: 6e 74 65 6e 74 6c 65 73 73 5f 64 65 6c 65 74 65 ntentless_delete +| 4080: 3d 31 2c 20 74 6f 6b 65 6e 64 61 74 61 3d 31 29 =1, tokendata=1) +| page 2 offset 4096 +| 0: 0d 0f eb 00 03 0e 17 00 0f e2 0e 17 0e 31 00 00 .............1.. +| 16: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3600: 00 00 00 00 00 00 00 18 0a 03 00 36 00 00 00 00 ...........6.... +| 3616: ff 00 00 01 01 01 01 00 01 01 01 01 01 01 00 00 ................ +| 3632: 07 83 29 84 80 80 80 80 01 04 00 86 56 00 00 01 ..).........V... +| 3648: 96 04 30 61 61 61 01 02 02 01 04 02 04 01 08 02 ..0aaa.......... +| 3664: 04 04 04 01 10 02 04 04 04 04 04 04 04 01 20 02 .............. . +| 3680: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 01 ................ +| 3696: 40 02 04 04 04 04 04 04 04 04 04 04 04 04 04 04 @............... +| 3712: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3728: 04 01 81 00 02 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3744: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3760: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3776: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3792: 04 04 04 04 02 02 62 63 01 06 01 01 02 01 03 62 ......bc.......b +| 3808: 62 62 02 02 03 01 04 03 06 01 08 03 06 06 06 01 bb.............. +| 3824: 10 03 06 06 06 06 06 06 06 01 20 03 06 06 06 06 .......... ..... +| 3840: 06 06 06 06 06 06 06 06 06 06 06 01 40 03 06 06 ............@... +| 3856: 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 ................ +| 3872: 06 06 06 06 06 06 06 06 06 06 16 06 06 02 02 63 ...............c +| 3888: 64 02 06 01 01 02 01 03 63 63 63 03 02 05 01 04 d.......ccc..... +| 3904: 05 0a 01 08 05 0a 0a 0a 01 10 05 0a 0a 0a 0a 0a ................ +| 3920: 0a 0a 01 20 05 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a ... ............ +| 3936: 0a 0a 0a 0a 02 02 64 65 03 06 01 01 02 01 03 64 ......de.......d +| 3952: 64 64 04 02 09 01 04 09 12 01 08 09 12 12 12 01 dd.............. +| 3968: 10 09 12 12 12 12 12 12 12 02 02 65 66 04 06 01 ...........ef... +| 3984: 01 02 01 03 65 65 65 05 02 11 01 04 11 22 01 08 ....eee......... +| 4000: 11 22 22 22 02 02 66 67 05 06 01 01 02 01 03 66 ......fg.......f +| 4016: 56 66 06 02 21 01 04 21 42 02 02 67 68 06 06 01 Vf..!..!B..gh... +| 4032: 01 02 cb 03 67 67 67 07 02 41 02 02 68 69 07 06 ....ggg..A..hi.. +| 4048: 01 01 02 04 81 13 09 50 09 2e 09 1c 09 12 09 0c .......P........ +| 4064: 09 08 07 01 03 00 14 07 81 77 07 00 00 00 15 22 .........w...... +| 4080: 00 00 00 00 ff 00 00 01 00 00 00 00 00 00 05 0c ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 07 0f c8 00 0f f8 0f f0 0f e8 0f e0 ................ +| 16: 0f d8 0f d0 0f c8 00 00 00 00 00 00 00 00 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 06 07 04 00 10 09 7f 01 ................ +| 4048: 06 06 04 00 10 09 3f 01 06 05 04 00 10 09 1f 01 ......?......... +| 4064: 06 04 04 00 10 09 0f 01 06 03 04 00 10 09 07 01 ................ +| 4080: 06 02 04 00 10 09 03 01 06 01 04 00 10 09 01 01 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-d57c01958e48ab.db +}]} {} + +do_catchsql_test 8.1 { + SELECT rowid FROM t1('a* NOT ý‘') ; +} {0 {1 2 3 4 5 6 7}} sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/manifest b/manifest index d37c195d80..f6c5d3e862 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\shandling\sOOM\sand\sother\serrors\sin\sfts5\swhen\squerying\stokendata=1\stables. -D 2023-12-07T18:41:49.112 +C Fix\sa\snull-pointer\sdereference\sin\sfts5\stokendata=1\scode. +D 2023-12-07T19:08:25.647 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 7a459f96b283477063f83f40f687859c1264aff42769829ea1ef665f9b7342b8 +F ext/fts5/fts5_index.c 1193379f19168e8ccb4831d07355a700459a004bd8ca56c9255c8e2d515b813a F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -145,7 +145,7 @@ F ext/fts5/test/fts5corrupt.test b6d4034b682bb3387bc44c510c71b3c67d4349e4df13949 F ext/fts5/test/fts5corrupt2.test 99e7e23a58b4d89eb7167c6de1669cbc595cd3c79ab333e0eb56405473319e77 F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78 F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3 -F ext/fts5/test/fts5corrupt5.test 4f1b96f740a50faa0db1e7a9ff72ea20db5b83564cacc25c7ee23560f2bcb0c2 +F ext/fts5/test/fts5corrupt5.test f9dbed6785c38d891c0b947fd6e973e9c4eb44991e1322f8d7e5281a019276f2 F ext/fts5/test/fts5corrupt6.test bf8eeae07825b088b9665d9d8e4accbd8dc9bf3cb85b6c64cf6c9e18ccc420a4 F ext/fts5/test/fts5corrupt7.test 80ad7f683a8bda2404731bb77e8c3dbbb620c1f6cc583cca8239f6accd6338c0 F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d8c6b246944934a7a6e027b3f5b986fd64a19dd5c5c5175f4ea8586da59a6764 -R cf423a1f60ce55f260a5bafd492cd4f3 +P bc911ab5953532956510c199be72b1d3c556f2d0ddbd7fc0ae6f5f917b337b48 +R 75b0f8b88efb2078e2ea2aad5e87b3ae U dan -Z 7283dc8d98cf6fe28540e176c5956186 +Z 824fa07c60b09c8dc172e4af722aed18 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9849120d16..3806ce1ffe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bc911ab5953532956510c199be72b1d3c556f2d0ddbd7fc0ae6f5f917b337b48 \ No newline at end of file +d69fa8f0504887f968d9a190ecb889ddb40bb1b56d0d4479f9819c106aec719b \ No newline at end of file From 7c12666fe9072c369874bb577bc3a514d59ebf91 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 7 Dec 2023 20:46:40 +0000 Subject: [PATCH 07/80] Avoid an assert() failure when querying an fts5vocab table that accesses a tokendata=1 fts5 table with corrupt %_data records. FossilOrigin-Name: 386ba9e20423fb2f623d6adc9d3c310fb1b135f54a1dad15ef3b593d97886926 --- ext/fts5/fts5_index.c | 5 ++++- ext/fts5/test/fts5vocab2.test | 24 ++++++++++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 735d06c29f..902af552a6 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -7031,6 +7031,10 @@ int sqlite3Fts5IndexQuery( int bTokendata = pConfig->bTokendata; if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); + if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){ + bTokendata = 0; + } + /* Figure out which index to search and set iIdx accordingly. If this ** is a prefix query for which there is no prefix index, set iIdx to ** greater than pConfig->nPrefix to indicate that the query will be @@ -7042,7 +7046,6 @@ int sqlite3Fts5IndexQuery( ** for internal sanity checking by the integrity-check in debug ** mode only. */ #ifdef SQLITE_DEBUG - if( flags & FTS5INDEX_QUERY_NOTOKENDATA ) bTokendata = 0; if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; diff --git a/ext/fts5/test/fts5vocab2.test b/ext/fts5/test/fts5vocab2.test index 6f7aad329c..ecacc50dab 100644 --- a/ext/fts5/test/fts5vocab2.test +++ b/ext/fts5/test/fts5vocab2.test @@ -280,6 +280,30 @@ do_catchsql_test 5.2 { INSERT INTO t1 SELECT randomblob(3000) FROM v1 } {1 {query aborted}} +#------------------------------------------------------------------------- +reset_db +sqlite3_fts5_may_be_corrupt 1 + +do_execsql_test 6.0 { + BEGIN TRANSACTION; + CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1); + REPLACE INTO t1_data VALUES(1,X'03090009'); + REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101'); + REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808'); + REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909'); + REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808'); + COMMIT; +} + +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row'); +} + +do_catchsql_test 6.2 { + SELECT * FROM t3; +} {1 {database disk image is malformed}} + +sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/manifest b/manifest index f6c5d3e862..43fe098e6c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\snull-pointer\sdereference\sin\sfts5\stokendata=1\scode. -D 2023-12-07T19:08:25.647 +C Avoid\san\sassert()\sfailure\swhen\squerying\san\sfts5vocab\stable\sthat\saccesses\sa\stokendata=1\sfts5\stable\swith\scorrupt\s%_data\srecords. +D 2023-12-07T20:46:40.220 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 1193379f19168e8ccb4831d07355a700459a004bd8ca56c9255c8e2d515b813a +F ext/fts5/fts5_index.c 3b9f7245d5ff80770ea959030f33c9cb686b6e2c655babee81a849094376742f F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -238,7 +238,7 @@ F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db638 F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d9f005b5a80ccf0bc F ext/fts5/test/fts5version.test d6e5a5897550afeccc2f8531d87404dc1c289ee89385dd4318dbdd75e71d7a67 F ext/fts5/test/fts5vocab.test 7ed80d9af1ddaaa1637da05e406327b5aac250848bc604c1c1cc667908b87760 -F ext/fts5/test/fts5vocab2.test 681980e92e031c9f3fe8d9c149189e876c108da2fb0fb3a25bd8a9b94bff8f68 +F ext/fts5/test/fts5vocab2.test 1b1f0059f762ffb404213d35dac013e010621f08128589b6ec7bec59d9a710f3 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 F ext/fts5/tool/fts5txt2db.tcl c0d43c8590656f8240e622b00957b3a0facc49482411a9fdc2870b45c0c82f9f F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bc911ab5953532956510c199be72b1d3c556f2d0ddbd7fc0ae6f5f917b337b48 -R 75b0f8b88efb2078e2ea2aad5e87b3ae +P d69fa8f0504887f968d9a190ecb889ddb40bb1b56d0d4479f9819c106aec719b +R 296885500e216c301696a13ea32c2874 U dan -Z 824fa07c60b09c8dc172e4af722aed18 +Z 332eede4eabfd6d41dcad2a894a2ef97 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3806ce1ffe..fdfae31f28 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d69fa8f0504887f968d9a190ecb889ddb40bb1b56d0d4479f9819c106aec719b \ No newline at end of file +386ba9e20423fb2f623d6adc9d3c310fb1b135f54a1dad15ef3b593d97886926 \ No newline at end of file From 0b427eb796537c3576aa218a45178ba3087771c4 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 7 Dec 2023 21:09:33 +0000 Subject: [PATCH 08/80] Ensure an fts5vocab table never uses a special tokendata=1 merge cursor. FossilOrigin-Name: 1e26510e83b40c9bd2e8bfa2a0e81f2cb915e78fed773204ef537683e48b61dc --- ext/fts5/fts5_vocab.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index d738ada311..4782d0fb94 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -629,7 +629,7 @@ static int fts5VocabFilterMethod( if( pEq ){ zTerm = (const char *)sqlite3_value_text(pEq); nTerm = sqlite3_value_bytes(pEq); - f = 0; + f = FTS5INDEX_QUERY_NOTOKENDATA; }else{ if( pGe ){ zTerm = (const char *)sqlite3_value_text(pGe); diff --git a/manifest b/manifest index 43fe098e6c..f0af2731f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\san\sassert()\sfailure\swhen\squerying\san\sfts5vocab\stable\sthat\saccesses\sa\stokendata=1\sfts5\stable\swith\scorrupt\s%_data\srecords. -D 2023-12-07T20:46:40.220 +C Ensure\san\sfts5vocab\stable\snever\suses\sa\sspecial\stokendata=1\smerge\scursor. +D 2023-12-07T21:09:33.888 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -105,7 +105,7 @@ F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13 F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80 -F ext/fts5/fts5_vocab.c aed56169ae5c1aa9b8189c779ffeef04ed516d3c712c06914e6d91a6759f4e4a +F ext/fts5/fts5_vocab.c 209e0c151e108d5f3621fa24b91e9b02f3750ee6c3f9ccec312df39481b68a09 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl 8b1848ac2baad10e444e4183034a52050b52d20b3796d9d30e78f01ab0d05583 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d69fa8f0504887f968d9a190ecb889ddb40bb1b56d0d4479f9819c106aec719b -R 296885500e216c301696a13ea32c2874 +P 386ba9e20423fb2f623d6adc9d3c310fb1b135f54a1dad15ef3b593d97886926 +R 9e40df16d98c21b180a16f29a22a3539 U dan -Z 332eede4eabfd6d41dcad2a894a2ef97 +Z 83c9845e856bbb23547304df7a116f92 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fdfae31f28..f6bf381ce3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -386ba9e20423fb2f623d6adc9d3c310fb1b135f54a1dad15ef3b593d97886926 \ No newline at end of file +1e26510e83b40c9bd2e8bfa2a0e81f2cb915e78fed773204ef537683e48b61dc \ No newline at end of file From 05980f59312fdb9b51a3d1717c4f4d22195b6328 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 8 Dec 2023 12:04:32 +0000 Subject: [PATCH 09/80] Avoid dropping an error code in new fts5 tokendata=1 code. FossilOrigin-Name: a66596e33dc9aa4bab2ec3ff45546e1321d0a11bdc764f8381b315292ca92423 --- ext/fts5/fts5_index.c | 1 + ext/fts5/test/fts5faultH.test | 48 +++++++++++++++++++++++++++++++++++ manifest | 14 +++++----- manifest.uuid | 2 +- 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 902af552a6..8a77523f6d 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6940,6 +6940,7 @@ static Fts5Iter *fts5SetupTokendataIter( pNewIter++; if( pPrevIter ) pPrevIter++; + if( p->rc ) break; } } fts5TokendataSetTermIfEof(pPrev, pSmall); diff --git a/ext/fts5/test/fts5faultH.test b/ext/fts5/test/fts5faultH.test index 9dd4cac0d6..540b889f37 100644 --- a/ext/fts5/test/fts5faultH.test +++ b/ext/fts5/test/fts5faultH.test @@ -88,6 +88,54 @@ do_faultsim_test 2 -faults oom* -prep { faultsim_test_result {0 {10 24}} } +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + + INSERT INTO t1(rowid, x) VALUES(9, 'bbb Bbb BBB'); + BEGIN; + INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(11, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(12, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(13, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(14, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(15, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(16, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(17, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(18, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(19, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(20, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(21, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(22, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(23, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(24, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(25, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(26, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(27, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(28, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(29, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(30, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(31, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(32, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(33, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB'); + COMMIT; +} + +do_faultsim_test 3 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1('BBB AND AAA'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {10 35}} +} finish_test diff --git a/manifest b/manifest index f0af2731f2..94c4bf3122 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\san\sfts5vocab\stable\snever\suses\sa\sspecial\stokendata=1\smerge\scursor. -D 2023-12-07T21:09:33.888 +C Avoid\sdropping\san\serror\scode\sin\snew\sfts5\stokendata=1\scode. +D 2023-12-08T12:04:32.576 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 3b9f7245d5ff80770ea959030f33c9cb686b6e2c655babee81a849094376742f +F ext/fts5/fts5_index.c ed206045ff0f2226d870fa41fba45f738c0cc953ab74ba68477091b9a574ccd3 F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -170,7 +170,7 @@ F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075 F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test d2e5a4d9a34e08dcaadcaeafef74d10cbc2abdd11aa2659a18af0294bf2812d3 -F ext/fts5/test/fts5faultH.test d845f45dac3e1a3f20c7e0a2be95280c95d3204c06802f86ab2c110e52ed3d14 +F ext/fts5/test/fts5faultH.test 57f53c87ffd59be0265840f2b54a16811f9cb9012db86aad9b41d0d14d85dfe3 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 386ba9e20423fb2f623d6adc9d3c310fb1b135f54a1dad15ef3b593d97886926 -R 9e40df16d98c21b180a16f29a22a3539 +P 1e26510e83b40c9bd2e8bfa2a0e81f2cb915e78fed773204ef537683e48b61dc +R 12765af39a039a08d248c249f36327f1 U dan -Z 83c9845e856bbb23547304df7a116f92 +Z 02f820ebfff0cef7481a1d5e262c99ab # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f6bf381ce3..dda7aa4a5b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e26510e83b40c9bd2e8bfa2a0e81f2cb915e78fed773204ef537683e48b61dc \ No newline at end of file +a66596e33dc9aa4bab2ec3ff45546e1321d0a11bdc764f8381b315292ca92423 \ No newline at end of file From 9d2446dc13ff564d7938ed8fb4d454660378f830 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 8 Dec 2023 12:58:41 +0000 Subject: [PATCH 10/80] Fix a harmless compiler warning about "confusing indentation". FossilOrigin-Name: 34f9e9a8c4bea13f60f43062e25cd7d9422f2e7f5b371ed0ddadc9abeb3ca256 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 94c4bf3122..6bcc2c1a68 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sdropping\san\serror\scode\sin\snew\sfts5\stokendata=1\scode. -D 2023-12-08T12:04:32.576 +C Fix\sa\sharmless\scompiler\swarning\sabout\s"confusing\sindentation". +D 2023-12-08T12:58:41.690 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -821,7 +821,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 1fdc69ce1333e9bd6d7d3df9fa5af1373a3f5bfdd52108d1dbc0ca85a55f777e +F src/where.c dad4b472b8dc886c0962409e70978568c06b5108bcf3dcd4938a9305a3540440 F src/whereInt.h 4b38c5889514e3aead3f27d0ee9a26e47c3f150efc59e2a8b4e3bc8835e4d7a1 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1e26510e83b40c9bd2e8bfa2a0e81f2cb915e78fed773204ef537683e48b61dc -R 12765af39a039a08d248c249f36327f1 -U dan -Z 02f820ebfff0cef7481a1d5e262c99ab +P a66596e33dc9aa4bab2ec3ff45546e1321d0a11bdc764f8381b315292ca92423 +R fedc3377b13a718cf2bf1e90affee545 +U drh +Z 0b5b9f54c9d3f54a21578c1d32159a58 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dda7aa4a5b..1acd6ed7f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a66596e33dc9aa4bab2ec3ff45546e1321d0a11bdc764f8381b315292ca92423 \ No newline at end of file +34f9e9a8c4bea13f60f43062e25cd7d9422f2e7f5b371ed0ddadc9abeb3ca256 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 83f979d760..c98f92d5a8 100644 --- a/src/where.c +++ b/src/where.c @@ -2020,7 +2020,8 @@ static int whereRangeScanEst( ** sample, then assume they are 4x more selective. This brings ** the estimated selectivity more in line with what it would be ** if estimated without the use of STAT4 tables. */ - if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); + if( iLwrIdx==iUprIdx ){ nNew -= 20; } + assert( 20==sqlite3LogEst(4) ); }else{ nNew = 10; assert( 10==sqlite3LogEst(2) ); } From 2a27be21078dfcd5e53827eee648d2fdfd3be330 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 8 Dec 2023 14:54:22 +0000 Subject: [PATCH 11/80] Fix a potential problem RCStr access on a JsonString object that is not really and RCStr. Fuzzer/UBSAN find. FossilOrigin-Name: d2f2174ce2cc89606034e158149a2d05fc3627ec4d5cdb772add7a2250f29d78 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6bcc2c1a68..55eab22459 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning\sabout\s"confusing\sindentation". -D 2023-12-08T12:58:41.690 +C Fix\sa\spotential\sproblem\sRCStr\saccess\son\sa\sJsonString\sobject\sthat\sis\snot\nreally\sand\sRCStr.\s\sFuzzer/UBSAN\sfind. +D 2023-12-08T14:54:22.333 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 3d64e42894afcda2efe8b112e8e2accd66fbeb150d7b1a073f20fc4a232faa37 +F src/json.c 0b92947cc3762e10f0c9afb5b3f42df3ac60bf0bae3ceb9751a0f0f5c3cab219 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a66596e33dc9aa4bab2ec3ff45546e1321d0a11bdc764f8381b315292ca92423 -R fedc3377b13a718cf2bf1e90affee545 +P 34f9e9a8c4bea13f60f43062e25cd7d9422f2e7f5b371ed0ddadc9abeb3ca256 +R 5b24ba3b196cb17162e42a315ae717a9 U drh -Z 0b5b9f54c9d3f54a21578c1d32159a58 +Z 497d51b892f444b107e833dd38c8bfdf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1acd6ed7f7..a3c777572a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34f9e9a8c4bea13f60f43062e25cd7d9422f2e7f5b371ed0ddadc9abeb3ca256 \ No newline at end of file +d2f2174ce2cc89606034e158149a2d05fc3627ec4d5cdb772add7a2250f29d78 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6a4a0babfc..5060801749 100644 --- a/src/json.c +++ b/src/json.c @@ -4115,7 +4115,7 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); if( isFinal ){ - sqlite3RCStrUnref(pStr->zBuf); + if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); }else{ pStr->nUsed--; } @@ -4235,7 +4235,7 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); if( isFinal ){ - sqlite3RCStrUnref(pStr->zBuf); + if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); }else{ pStr->nUsed--; } From bfa0de86e6a530550bb77cc4ff1e751b0d6d3cbb Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 8 Dec 2023 16:56:50 +0000 Subject: [PATCH 12/80] Fix a harmless UBSAN warning. FossilOrigin-Name: 1503cba6d17e9bade7a5c103ddd23241ff4741f9a2e3032ffe2987af243dae65 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 55eab22459..309f6fb6d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sproblem\sRCStr\saccess\son\sa\sJsonString\sobject\sthat\sis\snot\nreally\sand\sRCStr.\s\sFuzzer/UBSAN\sfind. -D 2023-12-08T14:54:22.333 +C Fix\sa\sharmless\sUBSAN\swarning. +D 2023-12-08T16:56:50.764 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 0b92947cc3762e10f0c9afb5b3f42df3ac60bf0bae3ceb9751a0f0f5c3cab219 +F src/json.c 5631fd482c13680f142cc9f3403022232584978fa9d0e86ac282c310bc3d8902 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 34f9e9a8c4bea13f60f43062e25cd7d9422f2e7f5b371ed0ddadc9abeb3ca256 -R 5b24ba3b196cb17162e42a315ae717a9 +P d2f2174ce2cc89606034e158149a2d05fc3627ec4d5cdb772add7a2250f29d78 +R 4e2efed7bb86322a932c64421cda8002 U drh -Z 497d51b892f444b107e833dd38c8bfdf +Z d8061dc139c0b2205f580742f9475aee # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a3c777572a..5eecdc8891 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2f2174ce2cc89606034e158149a2d05fc3627ec4d5cdb772add7a2250f29d78 \ No newline at end of file +1503cba6d17e9bade7a5c103ddd23241ff4741f9a2e3032ffe2987af243dae65 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 5060801749..6bb13941d9 100644 --- a/src/json.c +++ b/src/json.c @@ -1797,7 +1797,7 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ *pSz = 0; return 0; } - sz = (pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + + sz = ((u32)pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; n = 5; }else{ From b89e64d82231c3be84c5b38dd07a0ebf347ccbcb Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Dec 2023 02:39:11 +0000 Subject: [PATCH 13/80] Fix a potential use of uninitialized value in json_valid() with 2nd argument of 8. FossilOrigin-Name: fa102036fe46eeb71b7df3e265be1935ae5c78e0b939b08841bcfb8abadbc77a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 309f6fb6d3..f1e356acec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\sUBSAN\swarning. -D 2023-12-08T16:56:50.764 +C Fix\sa\spotential\suse\sof\suninitialized\svalue\sin\sjson_valid()\swith\s2nd\nargument\sof\s8. +D 2023-12-11T02:39:11.053 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 5631fd482c13680f142cc9f3403022232584978fa9d0e86ac282c310bc3d8902 +F src/json.c 022548736478f10bec4f2766f2c9fe1cc25c06b14cabb0ca64b4a1beb83fbf16 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d2f2174ce2cc89606034e158149a2d05fc3627ec4d5cdb772add7a2250f29d78 -R 4e2efed7bb86322a932c64421cda8002 +P 1503cba6d17e9bade7a5c103ddd23241ff4741f9a2e3032ffe2987af243dae65 +R 10c5202c73d5a29cc977b618b1930c91 U drh -Z d8061dc139c0b2205f580742f9475aee +Z 03628577bea44fb563932e72818a5e19 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5eecdc8891..c4f83fcd22 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1503cba6d17e9bade7a5c103ddd23241ff4741f9a2e3032ffe2987af243dae65 \ No newline at end of file +fa102036fe46eeb71b7df3e265be1935ae5c78e0b939b08841bcfb8abadbc77a \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6bb13941d9..71a1b789a4 100644 --- a/src/json.c +++ b/src/json.c @@ -3972,6 +3972,7 @@ static void jsonValidFunc( if( sx.eErr & JSTRING_OOM ) oom = 1; if( sx.eErr==0 ){ memset(&px, 0, sizeof(px)); + jsonStringTerminate(&sx); px.zJson = sx.zBuf; px.nJson = sx.nUsed; if( jsonXlateTextToBlob(&px, 0)==px.nJson ){ From ce46e0eb11ea5a518034d8a8415bd1d50da6dba5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Dec 2023 14:01:38 +0000 Subject: [PATCH 14/80] Work toward enhanced functionality for json_valid() with deep checking of the JSONB (second argument has bit 0x08). FossilOrigin-Name: c370d573198b151767f04e91bf8baa4ae0076751ae468c5709742a0b0ed16770 --- manifest | 15 +++-- manifest.uuid | 2 +- src/json.c | 149 ++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 136 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index f1e356acec..0ee7bf7d35 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\suse\sof\suninitialized\svalue\sin\sjson_valid()\swith\s2nd\nargument\sof\s8. -D 2023-12-11T02:39:11.053 +C Work\stoward\senhanced\sfunctionality\sfor\sjson_valid()\swith\sdeep\schecking\nof\sthe\sJSONB\s(second\sargument\shas\sbit\s0x08). +D 2023-12-11T14:01:38.458 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 022548736478f10bec4f2766f2c9fe1cc25c06b14cabb0ca64b4a1beb83fbf16 +F src/json.c 683a85f8b35a79c817b05723cee14ce7281a9f76e78d64950cfb50be10caf6d0 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2153,8 +2153,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1503cba6d17e9bade7a5c103ddd23241ff4741f9a2e3032ffe2987af243dae65 -R 10c5202c73d5a29cc977b618b1930c91 +P fa102036fe46eeb71b7df3e265be1935ae5c78e0b939b08841bcfb8abadbc77a +R d66c4373e6cbfb5a424f82330e7184cd +T *branch * jsonb-valid +T *sym-jsonb-valid * +T -sym-trunk * U drh -Z 03628577bea44fb563932e72818a5e19 +Z 18d49697fdb46dc3745b2a8b7db33d7b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c4f83fcd22..b365e86e8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa102036fe46eeb71b7df3e265be1935ae5c78e0b939b08841bcfb8abadbc77a \ No newline at end of file +c370d573198b151767f04e91bf8baa4ae0076751ae468c5709742a0b0ed16770 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 71a1b789a4..ab69f33ce0 100644 --- a/src/json.c +++ b/src/json.c @@ -3869,6 +3869,128 @@ json_type_done: jsonParseFree(p); } +/* +** Check a single element of the JSONB in pParse for validity. +** +** The element to be checked starts at offset i and must end at on the +** last byte before iEnd. +** +** Return 0 if everything is correct. Return the 1-based byte offset of the +** error if a problem is detected. (In other words, if the error is at offset +** 0, return 1). +*/ +static u32 jsonbValidityCheck(JsonParse *pParse, u32 i, u32 iEnd, u32 iDepth){ + u32 n, sz, j, k; + const u8 *z; + u8 x; + if( iDepth>JSON_MAX_DEPTH ) return i+1; + sz = 0; + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ) return i+1; + if( i+n+sz!=iEnd ) return i+1; + z = pParse->aBlob; + x = z[i] & 0x0f; + switch( x ){ + case JSONB_NULL: + case JSONB_TRUE: + case JSONB_FALSE: { + return n+sz==1 ? 0 : i+1; + } + default: { + return i+1; + } + case JSONB_INT: { + if( sz<1 ) return i+1; + j = i+n; + if( z[j]=='-' ){ + j++; + if( sz<2 ) return j; + } + k = i+n+sz; + while( jk ) return j+1; + sub = jsonbValidityCheck(pParse, j, k, iDepth+1); + if( sub ) return sub; + j += n + sz; + } + assert( j==k ); + return 0; + } + case JSONB_OBJECT: { + u32 cnt = 0; + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + if( (cnt & 1)==0 ){ + x = z[j] & 0x0f; + if( xJSONB_TEXTRAW ) return j+1; + } + sub = jsonbValidityCheck(pParse, j, k, iDepth+1); + if( sub ) return sub; + cnt++; + j += n + sz; + } + assert( j==k ); + if( (cnt & 1)!=0 ) return j+1; + return 0; + } + } +} + /* ** json_valid(JSON) ** json_valid(JSON, FLAGS) @@ -3954,38 +4076,19 @@ static void jsonValidFunc( case SQLITE_BLOB: { if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){ if( flags & 0x04 ){ - /* Superficial checking only - accomplisehd by the + /* Superficial checking only - accomplished by the ** jsonFuncArgMightBeBinary() call above. */ res = 1; }else{ /* Strict checking. Check by translating BLOB->TEXT->BLOB. If ** no errors occur, call that a "strict check". */ JsonParse px; - JsonString sx; - u8 oom = 0; + u32 iErr; memset(&px, 0, sizeof(px)); px.aBlob = (u8*)sqlite3_value_blob(argv[0]); px.nBlob = sqlite3_value_bytes(argv[0]); - jsonStringInit(&sx, 0); - jsonXlateBlobToText(&px, 0, &sx); - jsonParseReset(&px); - if( sx.eErr & JSTRING_OOM ) oom = 1; - if( sx.eErr==0 ){ - memset(&px, 0, sizeof(px)); - jsonStringTerminate(&sx); - px.zJson = sx.zBuf; - px.nJson = sx.nUsed; - if( jsonXlateTextToBlob(&px, 0)==px.nJson ){ - res = 1; - } - oom |= px.oom; - jsonParseReset(&px); - } - jsonStringReset(&sx); - if( oom ){ - sqlite3_result_error_nomem(ctx); - return; - } + iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); + res = iErr==0; } } break; From 7d2eaae83e46ef5f41419bdd4934f7ed4217529b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Dec 2023 17:03:12 +0000 Subject: [PATCH 15/80] Add SQLITE_TESTCTRL_VALIDATE_JSONB, which if enabled under SQLITE_DEBUG causes cross-checking of generate JSONB. FossilOrigin-Name: b410a4db74a650003539ffaaea18519d5159b504daac47db6a4874b730f40ac8 --- manifest | 25 ++--- manifest.uuid | 2 +- src/global.c | 3 + src/json.c | 288 ++++++++++++++++++++++++++++-------------------- src/main.c | 13 +++ src/shell.c.in | 7 ++ src/sqlite.h.in | 1 + src/sqliteInt.h | 3 + 8 files changed, 205 insertions(+), 137 deletions(-) diff --git a/manifest b/manifest index 0ee7bf7d35..2b659538e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\stoward\senhanced\sfunctionality\sfor\sjson_valid()\swith\sdeep\schecking\nof\sthe\sJSONB\s(second\sargument\shas\sbit\s0x08). -D 2023-12-11T14:01:38.458 +C Add\sSQLITE_TESTCTRL_VALIDATE_JSONB,\swhich\sif\senabled\sunder\sSQLITE_DEBUG\scauses\ncross-checking\sof\sgenerate\sJSONB. +D 2023-12-11T17:03:12.559 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -690,16 +690,16 @@ F src/expr.c 05278def9c186b5875d6903ea26148c7461b9ce0344f0fd7be9a0dfea0a4538a F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 -F src/global.c 29f56a330ed9d1b5cd9b79ac0ca36f97ac3afc730ff8bfa987b0db9e559d684d +F src/global.c c32438e08249ca9440cccff110283bc93b65d05320c8b162e492e8aef865d15c F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 683a85f8b35a79c817b05723cee14ce7281a9f76e78d64950cfb50be10caf6d0 +F src/json.c ea4b13e85f81a61d0331cec790e55648785864f31eba51a74b7190e895a37ac5 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 -F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 +F src/main.c f8bfeb360386291b1f5b9ab216ab8d8655bdb52c977bc8287db3fbfd3f8a2611 F src/malloc.c f016922435dc7d1f1f5083a03338a3e91f8c67ce2c5bdcfa4cdef62e612f5fcc F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 @@ -737,11 +737,11 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445 -F src/shell.c.in 9b6c3e641de45651ad0b5e9c26cd2f72efabee28179a5315d15c54239515ee3a -F src/sqlite.h.in d93a4821d2f792467a60f7dc81268d1bb8634f40c31694ef254cab4f9921f96a +F src/shell.c.in 64feb9fdb6829f7643b4e64e83f3b6261331474c4876abb8ebe70cfd33f25086 +F src/sqlite.h.in d6dee6231a2b248f6f2453bf9832a51694d90578f096bcfe49414c2d0211b6eb F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 39b6fe0652c763d8ff6bd0a2427c009e5abe05fb70f5a0cb145a34bcc614fb90 +F src/sqliteInt.h f34c338e2c34e4f0795f8110ecb64ad90252306c7b3f0587ad6cf82f68e7c401 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2153,11 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fa102036fe46eeb71b7df3e265be1935ae5c78e0b939b08841bcfb8abadbc77a -R d66c4373e6cbfb5a424f82330e7184cd -T *branch * jsonb-valid -T *sym-jsonb-valid * -T -sym-trunk * +P c370d573198b151767f04e91bf8baa4ae0076751ae468c5709742a0b0ed16770 +R c7157327882c9dc5aa9e50e7e992741a U drh -Z 18d49697fdb46dc3745b2a8b7db33d7b +Z 3bfa19b437a7164afac8549d2bca446e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b365e86e8e..a11be5a493 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c370d573198b151767f04e91bf8baa4ae0076751ae468c5709742a0b0ed16770 \ No newline at end of file +b410a4db74a650003539ffaaea18519d5159b504daac47db6a4874b730f40ac8 \ No newline at end of file diff --git a/src/global.c b/src/global.c index 60cd13e2ab..5db5565bf9 100644 --- a/src/global.c +++ b/src/global.c @@ -244,6 +244,9 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* bSmallMalloc */ 1, /* bExtraSchemaChecks */ sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */ +#ifdef SQLITE_DEBUG + 0, /* bJsonbValidate */ +#endif 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ diff --git a/src/json.c b/src/json.c index ab69f33ce0..f800dc931b 100644 --- a/src/json.c +++ b/src/json.c @@ -336,6 +336,7 @@ static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static void jsonParseFree(JsonParse*); +static u32 jsonbPayloadSize(const JsonParse*, u32, u32*); /************************************************************************** ** Utility routines for dealing with JsonCache objects @@ -1228,6 +1229,166 @@ static int jsonIs4HexB(const char *z, int *pOp){ return 1; } + +/* +** Check a single element of the JSONB in pParse for validity. +** +** The element to be checked starts at offset i and must end at on the +** last byte before iEnd. +** +** Return 0 if everything is correct. Return the 1-based byte offset of the +** error if a problem is detected. (In other words, if the error is at offset +** 0, return 1). +*/ +static u32 jsonbValidityCheck(JsonParse *pParse, u32 i, u32 iEnd, u32 iDepth){ + u32 n, sz, j, k; + const u8 *z; + u8 x; + if( iDepth>JSON_MAX_DEPTH ) return i+1; + sz = 0; + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ) return i+1; + if( i+n+sz!=iEnd ) return i+1; + z = pParse->aBlob; + x = z[i] & 0x0f; + switch( x ){ + case JSONB_NULL: + case JSONB_TRUE: + case JSONB_FALSE: { + return n+sz==1 ? 0 : i+1; + } + default: { + return i+1; + } + case JSONB_INT: { + if( sz<1 ) return i+1; + j = i+n; + if( z[j]=='-' ){ + j++; + if( sz<2 ) return j; + } + k = i+n+sz; + while( jk ) return i+1; + if( z[j+1]!='.' ) return i+1; + j += 2; + seen = 1; + } + for(; j0 ) return i+1; + if( x==JSONB_FLOAT && (j==k-1 || !sqlite3Isdigit(z[j+1])) ){ + return i+1; + } + seen = 1; + continue; + } + if( z[j]=='e' || z[j]=='E' ){ + if( seen==2 ) return i+1; + if( j==k-1 ) return i+1; + if( z[j+1]=='+' || z[j+1]=='-' ){ + j++; + if( j==k-1 ) return i+1; + } + seen = 2; + continue; + } + return i+1; + } + return 0; + } + case JSONB_TEXT: { + return 0; + } + case JSONB_TEXTJ: + case JSONB_TEXT5: { + return 0; + } + case JSONB_TEXTRAW: { + return 0; + } + case JSONB_ARRAY: { + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1); + if( sub ) return sub; + j += n + sz; + } + assert( j==k ); + return 0; + } + case JSONB_OBJECT: { + u32 cnt = 0; + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + if( (cnt & 1)==0 ){ + x = z[j] & 0x0f; + if( xJSONB_TEXTRAW ) return j+1; + } + sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1); + if( sub ) return sub; + cnt++; + j += n + sz; + } + assert( j==k ); + if( (cnt & 1)!=0 ) return j+1; + return 0; + } + } +} + /* ** Translate a single element of JSON text at pParse->zJson[i] into ** its equivalent binary JSONB representation. Append the translation into @@ -1713,7 +1874,12 @@ static int jsonConvertTextToBlob( i = jsonXlateTextToBlob(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ +#ifdef SQLITE_DEBUG assert( pParse->iDepth==0 ); + if( sqlite3Config.bJsonbValidate ){ + assert( jsonbValidityCheck(pParse, 0, pParse->nBlob, 0)==0 ); + } +#endif while( jsonIsspace(zJson[i]) ) i++; if( zJson[i] ){ i += json5Whitespace(&zJson[i]); @@ -3869,128 +4035,6 @@ json_type_done: jsonParseFree(p); } -/* -** Check a single element of the JSONB in pParse for validity. -** -** The element to be checked starts at offset i and must end at on the -** last byte before iEnd. -** -** Return 0 if everything is correct. Return the 1-based byte offset of the -** error if a problem is detected. (In other words, if the error is at offset -** 0, return 1). -*/ -static u32 jsonbValidityCheck(JsonParse *pParse, u32 i, u32 iEnd, u32 iDepth){ - u32 n, sz, j, k; - const u8 *z; - u8 x; - if( iDepth>JSON_MAX_DEPTH ) return i+1; - sz = 0; - n = jsonbPayloadSize(pParse, i, &sz); - if( n==0 ) return i+1; - if( i+n+sz!=iEnd ) return i+1; - z = pParse->aBlob; - x = z[i] & 0x0f; - switch( x ){ - case JSONB_NULL: - case JSONB_TRUE: - case JSONB_FALSE: { - return n+sz==1 ? 0 : i+1; - } - default: { - return i+1; - } - case JSONB_INT: { - if( sz<1 ) return i+1; - j = i+n; - if( z[j]=='-' ){ - j++; - if( sz<2 ) return j; - } - k = i+n+sz; - while( jk ) return j+1; - sub = jsonbValidityCheck(pParse, j, k, iDepth+1); - if( sub ) return sub; - j += n + sz; - } - assert( j==k ); - return 0; - } - case JSONB_OBJECT: { - u32 cnt = 0; - u32 sub; - j = i+n; - k = j+sz; - while( jk ) return j+1; - if( (cnt & 1)==0 ){ - x = z[j] & 0x0f; - if( xJSONB_TEXTRAW ) return j+1; - } - sub = jsonbValidityCheck(pParse, j, k, iDepth+1); - if( sub ) return sub; - cnt++; - j += n + sz; - } - assert( j==k ); - if( (cnt & 1)!=0 ) return j+1; - return 0; - } - } -} - /* ** json_valid(JSON) ** json_valid(JSON, FLAGS) diff --git a/src/main.c b/src/main.c index 6acfdc325d..553e3dc2d8 100644 --- a/src/main.c +++ b/src/main.c @@ -4659,6 +4659,19 @@ int sqlite3_test_control(int op, ...){ break; } #endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_VALIDATE_JSONB, (u8)trueFalse); + ** + ** Activate or deactivate validation of JSONB that is generated from + ** text. Off by default, as the validation is slow. Validation is + ** only available if compiled using SQLITE_DEBUG. + */ + case SQLITE_TESTCTRL_VALIDATE_JSONB: { +#if defined(SQLITE_DEBUG) + sqlite3Config.bJsonbValidate = (u8)(va_arg(ap, int)&0xff); +#endif + break; + } } va_end(ap); #endif /* SQLITE_UNTESTABLE */ diff --git a/src/shell.c.in b/src/shell.c.in index 6d1312d0f8..af7b1b6cfc 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10770,6 +10770,7 @@ static int do_meta_command(char *zLine, ShellState *p){ {"sorter_mmap", SQLITE_TESTCTRL_SORTER_MMAP, 0, "NMAX" }, {"tune", SQLITE_TESTCTRL_TUNE, 1, "ID VALUE" }, {"uselongdouble", SQLITE_TESTCTRL_USELONGDOUBLE,0,"?BOOLEAN|\"default\"?"}, + {"validate_jsonb", SQLITE_TESTCTRL_VALIDATE_JSONB,0, "BOOLEAN" }, }; int testctrl = -1; int iCtrl = -1; @@ -10974,6 +10975,12 @@ static int do_meta_command(char *zLine, ShellState *p){ isOk = 3; } break; + case SQLITE_TESTCTRL_VALIDATE_JSONB: + if( nArg==3 ){ + sqlite3_test_control(testctrl, booleanValue(azArg[2])); + isOk = 3; + } + break; } } if( isOk==0 && iCtrl>=0 ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ce32b85b18..84af1c54e6 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8300,6 +8300,7 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ +#define SQLITE_TESTCTRL_VALIDATE_JSONB 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a0d5962005..f2846ca6d4 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4177,6 +4177,9 @@ struct Sqlite3Config { u8 bSmallMalloc; /* Avoid large memory allocations if true */ u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ u8 bUseLongDouble; /* Make use of long double */ +#ifdef SQLITE_DEBUG + u8 bJsonbValidate; /* Double-check JSONB parsing */ +#endif int mxStrlen; /* Maximum string length */ int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ From ba5505657f94d5eb6bb7f0bd294e0c0b35b50dae Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Dec 2023 19:00:44 +0000 Subject: [PATCH 16/80] Rename the new test-control to SQLITE_TESTCTRL_JSON_SELFCHECK. Make it so that the current value of the setting can be interrogated. FossilOrigin-Name: 7aff1d9a4cb17ecd5abab21ab032f35a78741dd641ddd8cbcc85fc4a81a0707d --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/global.c | 2 +- src/json.c | 2 +- src/main.c | 15 ++++++++++++--- src/shell.c.in | 12 ++++++++---- src/sqlite.h.in | 2 +- src/sqliteInt.h | 2 +- 8 files changed, 36 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 2b659538e1..f0bb7f952c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_TESTCTRL_VALIDATE_JSONB,\swhich\sif\senabled\sunder\sSQLITE_DEBUG\scauses\ncross-checking\sof\sgenerate\sJSONB. -D 2023-12-11T17:03:12.559 +C Rename\sthe\snew\stest-control\sto\sSQLITE_TESTCTRL_JSON_SELFCHECK.\s\sMake\sit\sso\nthat\sthe\scurrent\svalue\sof\sthe\ssetting\scan\sbe\sinterrogated. +D 2023-12-11T19:00:44.322 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -690,16 +690,16 @@ F src/expr.c 05278def9c186b5875d6903ea26148c7461b9ce0344f0fd7be9a0dfea0a4538a F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 -F src/global.c c32438e08249ca9440cccff110283bc93b65d05320c8b162e492e8aef865d15c +F src/global.c 765a0656d6cbf043cb272ff0ae38f39cc46713539ffe6793258ed3eb4b188b52 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c ea4b13e85f81a61d0331cec790e55648785864f31eba51a74b7190e895a37ac5 +F src/json.c 484486f07aa17757c2277f4d3eac4b6ea174532ad27cd68288ed35b0e546bd15 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 -F src/main.c f8bfeb360386291b1f5b9ab216ab8d8655bdb52c977bc8287db3fbfd3f8a2611 +F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 F src/malloc.c f016922435dc7d1f1f5083a03338a3e91f8c67ce2c5bdcfa4cdef62e612f5fcc F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 @@ -737,11 +737,11 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445 -F src/shell.c.in 64feb9fdb6829f7643b4e64e83f3b6261331474c4876abb8ebe70cfd33f25086 -F src/sqlite.h.in d6dee6231a2b248f6f2453bf9832a51694d90578f096bcfe49414c2d0211b6eb +F src/shell.c.in 0cd2ef4b3c814dded5436625ab664d9a973cbc4266a1768e3aa4cbf11bb41eca +F src/sqlite.h.in adcc7dbfeea1e69d6d487139a7e90db8a48fe998f3f5bb0f85c683e6a6fa68ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h f34c338e2c34e4f0795f8110ecb64ad90252306c7b3f0587ad6cf82f68e7c401 +F src/sqliteInt.h 615524fff083abf9353aa4a3a8adca2d32b3e61e014e62b1a5335c454a386783 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c370d573198b151767f04e91bf8baa4ae0076751ae468c5709742a0b0ed16770 -R c7157327882c9dc5aa9e50e7e992741a +P b410a4db74a650003539ffaaea18519d5159b504daac47db6a4874b730f40ac8 +R 2937e9cc9e961c36660161b1501df048 U drh -Z 3bfa19b437a7164afac8549d2bca446e +Z a13ee25229a871327250064e3f63946b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a11be5a493..d68fb2986c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b410a4db74a650003539ffaaea18519d5159b504daac47db6a4874b730f40ac8 \ No newline at end of file +7aff1d9a4cb17ecd5abab21ab032f35a78741dd641ddd8cbcc85fc4a81a0707d \ No newline at end of file diff --git a/src/global.c b/src/global.c index 5db5565bf9..7f27d91d15 100644 --- a/src/global.c +++ b/src/global.c @@ -245,7 +245,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 1, /* bExtraSchemaChecks */ sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */ #ifdef SQLITE_DEBUG - 0, /* bJsonbValidate */ + 0, /* bJsonSelfcheck */ #endif 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ diff --git a/src/json.c b/src/json.c index f800dc931b..407705c017 100644 --- a/src/json.c +++ b/src/json.c @@ -1876,7 +1876,7 @@ static int jsonConvertTextToBlob( if( i>0 ){ #ifdef SQLITE_DEBUG assert( pParse->iDepth==0 ); - if( sqlite3Config.bJsonbValidate ){ + if( sqlite3Config.bJsonSelfcheck ){ assert( jsonbValidityCheck(pParse, 0, pParse->nBlob, 0)==0 ); } #endif diff --git a/src/main.c b/src/main.c index 553e3dc2d8..50572ef87f 100644 --- a/src/main.c +++ b/src/main.c @@ -4660,15 +4660,24 @@ int sqlite3_test_control(int op, ...){ } #endif - /* sqlite3_test_control(SQLITE_TESTCTRL_VALIDATE_JSONB, (u8)trueFalse); + /* sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &onOff); ** ** Activate or deactivate validation of JSONB that is generated from ** text. Off by default, as the validation is slow. Validation is ** only available if compiled using SQLITE_DEBUG. + ** + ** If onOff is initially 1, then turn it on. If onOff is initially + ** off, turn it off. If onOff is initially -1, then change onOff + ** to be the current setting. */ - case SQLITE_TESTCTRL_VALIDATE_JSONB: { + case SQLITE_TESTCTRL_JSON_SELFCHECK: { #if defined(SQLITE_DEBUG) - sqlite3Config.bJsonbValidate = (u8)(va_arg(ap, int)&0xff); + int *pOnOff = va_arg(ap, int*); + if( *pOnOff<0 ){ + *pOnOff = sqlite3Config.bJsonSelfcheck; + }else{ + sqlite3Config.bJsonSelfcheck = (u8)((*pOnOff)&0xff); + } #endif break; } diff --git a/src/shell.c.in b/src/shell.c.in index af7b1b6cfc..35ed0cb5c4 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10756,6 +10756,7 @@ static int do_meta_command(char *zLine, ShellState *p){ {"fk_no_action", SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN" }, {"imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"}, {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" }, + {"json_selfcheck", SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN" }, {"localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN" }, {"never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN" }, {"optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK" }, @@ -10770,7 +10771,6 @@ static int do_meta_command(char *zLine, ShellState *p){ {"sorter_mmap", SQLITE_TESTCTRL_SORTER_MMAP, 0, "NMAX" }, {"tune", SQLITE_TESTCTRL_TUNE, 1, "ID VALUE" }, {"uselongdouble", SQLITE_TESTCTRL_USELONGDOUBLE,0,"?BOOLEAN|\"default\"?"}, - {"validate_jsonb", SQLITE_TESTCTRL_VALIDATE_JSONB,0, "BOOLEAN" }, }; int testctrl = -1; int iCtrl = -1; @@ -10975,11 +10975,15 @@ static int do_meta_command(char *zLine, ShellState *p){ isOk = 3; } break; - case SQLITE_TESTCTRL_VALIDATE_JSONB: - if( nArg==3 ){ - sqlite3_test_control(testctrl, booleanValue(azArg[2])); + case SQLITE_TESTCTRL_JSON_SELFCHECK: + if( nArg==2 ){ + rc2 = -1; + isOk = 1; + }else{ + rc2 = booleanValue(azArg[2]); isOk = 3; } + sqlite3_test_control(testctrl, &rc2); break; } } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 84af1c54e6..1f53438142 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8300,7 +8300,7 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ -#define SQLITE_TESTCTRL_VALIDATE_JSONB 14 +#define SQLITE_TESTCTRL_JSON_SELFCHECK 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f2846ca6d4..83226b5750 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4178,7 +4178,7 @@ struct Sqlite3Config { u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ u8 bUseLongDouble; /* Make use of long double */ #ifdef SQLITE_DEBUG - u8 bJsonbValidate; /* Double-check JSONB parsing */ + u8 bJsonSelfcheck; /* Double-check JSON parsing */ #endif int mxStrlen; /* Maximum string length */ int neverCorrupt; /* Database is always well-formed */ From 74456595595d753a6b6f3f42a0505f347064d129 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Dec 2023 19:21:06 +0000 Subject: [PATCH 17/80] Activate JSON_SELFCHECK within fuzzcheck. FossilOrigin-Name: 4d14e733bb521aed65e98533969d2303738232ae87dab70fdf7962e6513195f5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fuzzcheck.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f0bb7f952c..770abb234a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\sthe\snew\stest-control\sto\sSQLITE_TESTCTRL_JSON_SELFCHECK.\s\sMake\sit\sso\nthat\sthe\scurrent\svalue\sof\sthe\ssetting\scan\sbe\sinterrogated. -D 2023-12-11T19:00:44.322 +C Activate\sJSON_SELFCHECK\swithin\sfuzzcheck. +D 2023-12-11T19:21:06.298 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1229,7 +1229,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 69b8549e112fb815931a8c14c7955a0c407ae91a79356eecb82458384f2cb989 +F test/fuzzcheck.c 442b0b02146bb905c47c4ee1388949e7fd9d27acf22d9535e9abf216e582b956 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b410a4db74a650003539ffaaea18519d5159b504daac47db6a4874b730f40ac8 -R 2937e9cc9e961c36660161b1501df048 +P 7aff1d9a4cb17ecd5abab21ab032f35a78741dd641ddd8cbcc85fc4a81a0707d +R 73f673c74c17eaf64270ba7fabaefbeb U drh -Z a13ee25229a871327250064e3f63946b +Z 036937c49ad3c44a290e0f1c7a504e81 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d68fb2986c..c7f1e82f00 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7aff1d9a4cb17ecd5abab21ab032f35a78741dd641ddd8cbcc85fc4a81a0707d \ No newline at end of file +4d14e733bb521aed65e98533969d2303738232ae87dab70fdf7962e6513195f5 \ No newline at end of file diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 23200a5f03..d71335510d 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1267,6 +1267,8 @@ int runCombinedDbSqlInput( } sqlite3_limit(cx.db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 100); sqlite3_hard_heap_limit64(heapLimit); + rc = 1; + sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &rc); if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){ aDb[18] = aDb[19] = 1; From 001caa714fc29ca13aa86605aa2c18a47f875430 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Dec 2023 20:19:10 +0000 Subject: [PATCH 18/80] json_valid(*,8) allows minus-signs on hexadecimal literals. FossilOrigin-Name: c0d7f4520d839a268b3fd2474d0897a9832aa608bd6238b3e287fabecf07a350 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 770abb234a..29dc28440b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Activate\sJSON_SELFCHECK\swithin\sfuzzcheck. -D 2023-12-11T19:21:06.298 +C json_valid(*,8)\sallows\sminus-signs\son\shexadecimal\sliterals. +D 2023-12-11T20:19:10.982 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 484486f07aa17757c2277f4d3eac4b6ea174532ad27cd68288ed35b0e546bd15 +F src/json.c d0c018e1d9dba58c09518b15b4d0a1847bc8be7f7f050524618b486c8e286db8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7aff1d9a4cb17ecd5abab21ab032f35a78741dd641ddd8cbcc85fc4a81a0707d -R 73f673c74c17eaf64270ba7fabaefbeb +P 4d14e733bb521aed65e98533969d2303738232ae87dab70fdf7962e6513195f5 +R a882161352f3e912beb8ce222b601993 U drh -Z 036937c49ad3c44a290e0f1c7a504e81 +Z 09da734bc01fda75ed726fff4da7e465 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c7f1e82f00..c1e7de9ccc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d14e733bb521aed65e98533969d2303738232ae87dab70fdf7962e6513195f5 \ No newline at end of file +c0d7f4520d839a268b3fd2474d0897a9832aa608bd6238b3e287fabecf07a350 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 407705c017..b735987734 100644 --- a/src/json.c +++ b/src/json.c @@ -1280,6 +1280,10 @@ static u32 jsonbValidityCheck(JsonParse *pParse, u32 i, u32 iEnd, u32 iDepth){ case JSONB_INT5: { if( sz<3 ) return i+1; j = i+n; + if( z[j]=='-' ){ + if( sz<4 ) return i+1; + j++; + } if( z[j]!='0' ) return j+1; if( z[j+1]!='x' && z[j+1]!='X' ) return j+2; j += 2; From 5a890b4ed2b0f5732fa370dfbae295aa41748e0b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Dec 2023 20:44:21 +0000 Subject: [PATCH 19/80] json_error_position() now uses jsonValidityCheck() to find the approximate position of an error in a JSONB blob. FossilOrigin-Name: c3d60cf7028a333b825d5b89516945a73e0c158ac81d8bcc117d21bfd98602c8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 14 ++++---------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 29dc28440b..2e4f529d17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C json_valid(*,8)\sallows\sminus-signs\son\shexadecimal\sliterals. -D 2023-12-11T20:19:10.982 +C json_error_position()\snow\suses\sjsonValidityCheck()\sto\sfind\sthe\sapproximate\nposition\sof\san\serror\sin\sa\sJSONB\sblob. +D 2023-12-11T20:44:21.543 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c d0c018e1d9dba58c09518b15b4d0a1847bc8be7f7f050524618b486c8e286db8 +F src/json.c 191626c46afe3325a01c1a77b7aef6b10b8dc8a76c38757adeca89e51a4248db F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d14e733bb521aed65e98533969d2303738232ae87dab70fdf7962e6513195f5 -R a882161352f3e912beb8ce222b601993 +P c0d7f4520d839a268b3fd2474d0897a9832aa608bd6238b3e287fabecf07a350 +R 3b1676d0420fc714545bd48a8af4a295 U drh -Z 09da734bc01fda75ed726fff4da7e465 +Z 901bf4bbda4e9c3a1530e623c45e3a19 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c1e7de9ccc..825ac56d41 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0d7f4520d839a268b3fd2474d0897a9832aa608bd6238b3e287fabecf07a350 \ No newline at end of file +c3d60cf7028a333b825d5b89516945a73e0c158ac81d8bcc117d21bfd98602c8 \ No newline at end of file diff --git a/src/json.c b/src/json.c index b735987734..fd741f696d 100644 --- a/src/json.c +++ b/src/json.c @@ -4170,9 +4170,9 @@ static void jsonValidFunc( ** ** If the argument is NULL, return NULL ** -** If the argument is BLOB, do a fast validity check and return non-zero -** if the check fails. The returned value does not indicate where in the -** BLOB the error occurs. +** If the argument is BLOB, do a full validity check and return non-zero +** if the check fails. The return value is the approximate 1-based offset +** to the byte of the element that contains the first error. ** ** Otherwise interpret the argument is TEXT (even if it is numeric) and ** return the 1-based character position for where the parser first recognized @@ -4191,15 +4191,9 @@ static void jsonErrorFunc( UNUSED_PARAMETER(argc); memset(&s, 0, sizeof(s)); if( jsonFuncArgMightBeBinary(argv[0]) ){ - JsonString out; - jsonStringInit(&out, 0); s.aBlob = (u8*)sqlite3_value_blob(argv[0]); s.nBlob = sqlite3_value_bytes(argv[0]); - jsonXlateBlobToText(&s, 0, &out); - if( out.eErr ){ - iErrPos = (out.eErr & JSTRING_MALFORMED)!=0 ? 1 : -1; - } - jsonStringReset(&out); + iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 0); }else{ s.zJson = (char*)sqlite3_value_text(argv[0]); if( s.zJson==0 ) return; /* NULL input or OOM */ From ad6bc61804547bab9e358809a9281a2facdbadd8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 11 Dec 2023 21:00:55 +0000 Subject: [PATCH 20/80] The json_error_position() function now reports an approximate byte offset to the problem in a JSONB if there is a problem. FossilOrigin-Name: 80d5d94dff6a2d2557039be3d7d47c1a6003c4b98defe0bd411acfeb963ad5dd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 30 +++++++++++++++++------------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 2e4f529d17..ada03e02f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C json_error_position()\snow\suses\sjsonValidityCheck()\sto\sfind\sthe\sapproximate\nposition\sof\san\serror\sin\sa\sJSONB\sblob. -D 2023-12-11T20:44:21.543 +C The\sjson_error_position()\sfunction\snow\sreports\san\sapproximate\sbyte\soffset\nto\sthe\sproblem\sin\sa\sJSONB\sif\sthere\sis\sa\sproblem. +D 2023-12-11T21:00:55.202 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 191626c46afe3325a01c1a77b7aef6b10b8dc8a76c38757adeca89e51a4248db +F src/json.c 387dd9d7d63bac1e0068d66e8b40d86e1e2f0edc6930da28c67584c7a5a32872 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c0d7f4520d839a268b3fd2474d0897a9832aa608bd6238b3e287fabecf07a350 -R 3b1676d0420fc714545bd48a8af4a295 +P c3d60cf7028a333b825d5b89516945a73e0c158ac81d8bcc117d21bfd98602c8 +R eef20549bb0818034c7a80cb1490f371 U drh -Z 901bf4bbda4e9c3a1530e623c45e3a19 +Z 99dd825f602cc4286df6f3fcbe1570d6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 825ac56d41..07345f0192 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3d60cf7028a333b825d5b89516945a73e0c158ac81d8bcc117d21bfd98602c8 \ No newline at end of file +80d5d94dff6a2d2557039be3d7d47c1a6003c4b98defe0bd411acfeb963ad5dd \ No newline at end of file diff --git a/src/json.c b/src/json.c index fd741f696d..31367fe544 100644 --- a/src/json.c +++ b/src/json.c @@ -1240,7 +1240,12 @@ static int jsonIs4HexB(const char *z, int *pOp){ ** error if a problem is detected. (In other words, if the error is at offset ** 0, return 1). */ -static u32 jsonbValidityCheck(JsonParse *pParse, u32 i, u32 iEnd, u32 iDepth){ +static u32 jsonbValidityCheck( + const JsonParse *pParse, /* Input JSONB. Only aBlob and nBlob are used */ + u32 i, /* Start of element as pParse->aBlob[i] */ + u32 iEnd, /* One more than the last byte of the element */ + u32 iDepth /* Current nesting depth */ +){ u32 n, sz, j, k; const u8 *z; u8 x; @@ -1265,7 +1270,7 @@ static u32 jsonbValidityCheck(JsonParse *pParse, u32 i, u32 iEnd, u32 iDepth){ j = i+n; if( z[j]=='-' ){ j++; - if( sz<2 ) return j; + if( sz<2 ) return i+1; } k = i+n+sz; while( jk ) return i+1; - if( z[j+1]!='.' ) return i+1; - j += 2; - seen = 1; + if( j+3>k ) return j+1; + if( z[j+1]!='.' && z[j+1]!='e' && z[j+1]!='E' ) return j+1; + j++; } for(; j0 ) return i+1; if( x==JSONB_FLOAT && (j==k-1 || !sqlite3Isdigit(z[j+1])) ){ - return i+1; + return j+1; } seen = 1; continue; } if( z[j]=='e' || z[j]=='E' ){ - if( seen==2 ) return i+1; - if( j==k-1 ) return i+1; + if( seen==2 ) return j+1; + if( j==k-1 ) return j+1; if( z[j+1]=='+' || z[j+1]=='-' ){ j++; - if( j==k-1 ) return i+1; + if( j==k-1 ) return j+1; } seen = 2; continue; } - return i+1; + return j+1; } return 0; } From cc1a39fd247cf90af0ae59304206faf7ecb61819 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 12 Dec 2023 02:31:12 +0000 Subject: [PATCH 21/80] Validity checking of text nodes in JSONB. FossilOrigin-Name: fa5160687c2f970d407e8af73c246f7cd806bb4ce35f29a79ac534a8646a6c8e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ada03e02f1..91ba64c3dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sjson_error_position()\sfunction\snow\sreports\san\sapproximate\sbyte\soffset\nto\sthe\sproblem\sin\sa\sJSONB\sif\sthere\sis\sa\sproblem. -D 2023-12-11T21:00:55.202 +C Validity\schecking\sof\stext\snodes\sin\sJSONB. +D 2023-12-12T02:31:12.655 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 387dd9d7d63bac1e0068d66e8b40d86e1e2f0edc6930da28c67584c7a5a32872 +F src/json.c 34f829db190ae485926855da651d349eedb35fa56010266e14c0b094f536c9f5 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c3d60cf7028a333b825d5b89516945a73e0c158ac81d8bcc117d21bfd98602c8 -R eef20549bb0818034c7a80cb1490f371 +P 80d5d94dff6a2d2557039be3d7d47c1a6003c4b98defe0bd411acfeb963ad5dd +R 8ac47232b4f02631d44b831db0790bc1 U drh -Z 99dd825f602cc4286df6f3fcbe1570d6 +Z 8178cefe916c4486a65df71a751de097 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 07345f0192..208d3cd26d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -80d5d94dff6a2d2557039be3d7d47c1a6003c4b98defe0bd411acfeb963ad5dd \ No newline at end of file +fa5160687c2f970d407e8af73c246f7cd806bb4ce35f29a79ac534a8646a6c8e \ No newline at end of file diff --git a/src/json.c b/src/json.c index 31367fe544..17ebc769cc 100644 --- a/src/json.c +++ b/src/json.c @@ -176,7 +176,9 @@ static const char jsonSpaces[] = "\011\012\015\040"; /* ** Characters that are special to JSON. Control characters, -** '"' and '\\'. +** '"' and '\\' and '\''. Actually, '\'' is not special to +** canonical JSON, but it is special in JSON-5, so we include +** it in the set of special characters. */ static const char jsonIsOk[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -337,6 +339,7 @@ static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static void jsonParseFree(JsonParse*); static u32 jsonbPayloadSize(const JsonParse*, u32, u32*); +static u32 jsonUnescapeOneChar(const char*, u32, u32*); /************************************************************************** ** Utility routines for dealing with JsonCache objects @@ -1346,10 +1349,42 @@ static u32 jsonbValidityCheck( return 0; } case JSONB_TEXT: { + j = i+n; + k = j+sz; + while( j=k ){ + return j+1; + }else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){ + j++; + }else if( z[j+1]=='u' ){ + if( j+5>=k ) return j+1; + if( !jsonIs4Hex((const char*)&z[j+2]) ) return j+1; + j++; + }else if( x!=JSONB_TEXT5 ){ + return j+1; + }else{ + u32 c = 0; + u32 szC = jsonUnescapeOneChar((const char*)&z[j], k-j, &c); + if( c==0xfffd ) return j+1; + j += szC - 1; + } + } + j++; + } return 0; } case JSONB_TEXTRAW: { @@ -1625,6 +1660,8 @@ json_parse_restart: /* Control characters are not allowed in strings */ pParse->iErr = j; return -1; + }else if( c=='"' ){ + opcode = JSONB_TEXT5; } j++; } @@ -2084,13 +2121,13 @@ static u32 jsonXlateBlobToText( } break; } + case JSONB_TEXT: case JSONB_TEXTJ: { jsonAppendChar(pOut, '"'); jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); jsonAppendChar(pOut, '"'); break; } - case JSONB_TEXT: case JSONB_TEXT5: { const char *zIn; u32 k; From 87399a56db384f4ebf188652132ef0f29cd57b75 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 12 Dec 2023 14:33:52 +0000 Subject: [PATCH 22/80] Improvements to JSONB validation - catch more cases where the input does not conform to spec. FossilOrigin-Name: be1864eac4eb75cc30bf98f73092c8608467f4bd956240df6a0cbea9f1e09e85 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 12 +++++++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 91ba64c3dd..2e875799d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Validity\schecking\sof\stext\snodes\sin\sJSONB. -D 2023-12-12T02:31:12.655 +C Improvements\sto\sJSONB\svalidation\s-\scatch\smore\scases\swhere\sthe\sinput\sdoes\nnot\sconform\sto\sspec. +D 2023-12-12T14:33:52.419 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 34f829db190ae485926855da651d349eedb35fa56010266e14c0b094f536c9f5 +F src/json.c 6fc223a6cf1f475dfe72376018615ed85a1415b57121571ad65747f8b845b57f F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 80d5d94dff6a2d2557039be3d7d47c1a6003c4b98defe0bd411acfeb963ad5dd -R 8ac47232b4f02631d44b831db0790bc1 +P fa5160687c2f970d407e8af73c246f7cd806bb4ce35f29a79ac534a8646a6c8e +R 06eab0897027e01e7643ca8237160284 U drh -Z 8178cefe916c4486a65df71a751de097 +Z fe2966c205a693292efc9fe6aaf9e427 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 208d3cd26d..8f1ff9f03a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa5160687c2f970d407e8af73c246f7cd806bb4ce35f29a79ac534a8646a6c8e \ No newline at end of file +be1864eac4eb75cc30bf98f73092c8608467f4bd956240df6a0cbea9f1e09e85 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 17ebc769cc..5487f04022 100644 --- a/src/json.c +++ b/src/json.c @@ -1265,9 +1265,6 @@ static u32 jsonbValidityCheck( case JSONB_FALSE: { return n+sz==1 ? 0 : i+1; } - default: { - return i+1; - } case JSONB_INT: { if( sz<1 ) return i+1; j = i+n; @@ -1316,6 +1313,7 @@ static u32 jsonbValidityCheck( if( sz<3 ) return i+1; } if( z[j]=='.' ){ + if( x==JSONB_FLOAT ) return j+1; if( !sqlite3Isdigit(z[j+1]) ) return j+1; j += 2; seen = 1; @@ -1327,7 +1325,7 @@ static u32 jsonbValidityCheck( for(; j0 ) return i+1; + if( seen>0 ) return j+1; if( x==JSONB_FLOAT && (j==k-1 || !sqlite3Isdigit(z[j+1])) ){ return j+1; } @@ -1346,6 +1344,7 @@ static u32 jsonbValidityCheck( } return j+1; } + if( seen==0 ) return i+1; return 0; } case JSONB_TEXT: { @@ -1429,6 +1428,9 @@ static u32 jsonbValidityCheck( if( (cnt & 1)!=0 ) return j+1; return 0; } + default: { + return i+1; + } } } @@ -4234,7 +4236,7 @@ static void jsonErrorFunc( if( jsonFuncArgMightBeBinary(argv[0]) ){ s.aBlob = (u8*)sqlite3_value_blob(argv[0]); s.nBlob = sqlite3_value_bytes(argv[0]); - iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 0); + iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 1); }else{ s.zJson = (char*)sqlite3_value_text(argv[0]); if( s.zJson==0 ) return; /* NULL input or OOM */ From 78e636bba65e57b364263418bdde2417752bd9fc Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 12 Dec 2023 17:13:10 +0000 Subject: [PATCH 23/80] Add NEVER to two unreachable branches in JSON. FossilOrigin-Name: c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2e875799d2..2ef2ef6944 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sJSONB\svalidation\s-\scatch\smore\scases\swhere\sthe\sinput\sdoes\nnot\sconform\sto\sspec. -D 2023-12-12T14:33:52.419 +C Add\sNEVER\sto\stwo\sunreachable\sbranches\sin\sJSON. +D 2023-12-12T17:13:10.122 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6fc223a6cf1f475dfe72376018615ed85a1415b57121571ad65747f8b845b57f +F src/json.c cd270629b4b99d065522b45c3d2b5185491b1ab3544843c6f780aa50755d54dc F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fa5160687c2f970d407e8af73c246f7cd806bb4ce35f29a79ac534a8646a6c8e -R 06eab0897027e01e7643ca8237160284 +P be1864eac4eb75cc30bf98f73092c8608467f4bd956240df6a0cbea9f1e09e85 +R 6e221a94cd5525fe9f8a7e94ea467b60 U drh -Z fe2966c205a693292efc9fe6aaf9e427 +Z 2ea9e875faf360824d045f495f8867aa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8f1ff9f03a..838b8ad2ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be1864eac4eb75cc30bf98f73092c8608467f4bd956240df6a0cbea9f1e09e85 \ No newline at end of file +c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 5487f04022..e94cac6d52 100644 --- a/src/json.c +++ b/src/json.c @@ -1255,8 +1255,8 @@ static u32 jsonbValidityCheck( if( iDepth>JSON_MAX_DEPTH ) return i+1; sz = 0; n = jsonbPayloadSize(pParse, i, &sz); - if( n==0 ) return i+1; - if( i+n+sz!=iEnd ) return i+1; + if( NEVER(n==0) ) return i+1; /* Checked by caller */ + if( NEVER(i+n+sz!=iEnd) ) return i+1; /* Checked by caller */ z = pParse->aBlob; x = z[i] & 0x0f; switch( x ){ From 9710fa119e2d4196b1afbffb1f8b17659b0021f5 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 12 Dec 2023 17:52:27 +0000 Subject: [PATCH 24/80] Worker1 Promiser API: when multiple db connections are active then use the requested connection instead of always the first-opened connection. Bug reported in [forum:894c330e7f23b177|forum post 894c330e7f23b177]. FossilOrigin-Name: 194276e18e0268829061c09317e7f9f527a703eb45f1755ff1dd30bd99dc1b68 --- ext/wasm/api/sqlite3-worker1-promiser.c-pp.js | 3 ++- manifest | 15 +++++++-------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index 1f88b713a9..d4445cdcdf 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -199,10 +199,11 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi msg = Object.create(null); msg.type = arguments[0]; msg.args = arguments[1]; + msg.dbId = msg.args.dbId; }else{ toss("Invalid arugments for sqlite3Worker1Promiser()-created factory."); } - if(!msg.dbId) msg.dbId = dbId; + if(!msg.dbId && msg.type!=='open') msg.dbId = dbId; msg.messageId = genMsgId(msg); msg.departureTime = performance.now(); const proxy = Object.create(null); diff --git a/manifest b/manifest index 011acb7799..8840326af4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sjson_valid(*,8)\sfunction\sdoes\sa\smuch\sbetter\scheck\sof\sthe\svalidity\sof\nthe\sJSONB\sinput.\s\sThe\sjson_error_position()\sfunction\sreturns\san\sapproximate\nbyte\soffset\sto\sthe\spoint\sof\sthe\sfirst\sdetected\serror\sin\sthe\sJSONB. -D 2023-12-12T17:31:17.429 +C Worker1\sPromiser\sAPI:\swhen\smultiple\sdb\sconnections\sare\sactive\sthen\suse\sthe\srequested\sconnection\sinstead\sof\salways\sthe\sfirst-opened\sconnection.\sBug\sreported\sin\s[forum:894c330e7f23b177|forum\spost\s894c330e7f23b177]. +D 2023-12-12T17:52:27.818 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -602,7 +602,7 @@ F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 595953994aa3ae2287c889c4da39ab3d6f17b6461ecf4bec334b7a3faafddb02 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91ccfafc38b400fe36e90bdda88e F ext/wasm/api/sqlite3-wasm.c d0e09eb5ed3743c00294e30019e591c3aa150572ae7ffe8a8994568a7377589f -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 569d4e859968e65f55dec5fac0b879828a23c8b179162cc7812fec19f844dd21 +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js a94e3f0ca25a777bb73779368f97be0e103e02d067ad3ee3e9c4cc5bcefbd01c F ext/wasm/api/sqlite3-worker1.c-pp.js a541112aa51e16705f13a99bb943c64efe178aa28c86704a955f8fd9afe4ba37 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd @@ -2153,9 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fa102036fe46eeb71b7df3e265be1935ae5c78e0b939b08841bcfb8abadbc77a c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229 -R 6e221a94cd5525fe9f8a7e94ea467b60 -T +closed c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229 -U drh -Z 749a6c45182c15aa056f901d03cf2368 +P 840efb33a3934a9bdccad65ea15c70d5c335a517502c9844aaad8405d16d1604 +R 5c3d83f631ee20b7dd8bc56f0a1d21a8 +U stephan +Z 26c52cb9495b2f9f42b5f7d1f0a8e853 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5e0946fbbd..481d66b8fb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -840efb33a3934a9bdccad65ea15c70d5c335a517502c9844aaad8405d16d1604 \ No newline at end of file +194276e18e0268829061c09317e7f9f527a703eb45f1755ff1dd30bd99dc1b68 \ No newline at end of file From 891f1dc0542504fe402d1879651c307ae9355825 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 12 Dec 2023 18:38:53 +0000 Subject: [PATCH 25/80] Fix the JSON object label comparison object so that it works correctly even if the label ends with escaped whitespace. FossilOrigin-Name: 4d5353cadd7b7c5f105bc197f3ec739e2d041472d6b3e939654c9f9cfc2749ae --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 12 ++++++++---- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 8840326af4..ca6d324ceb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Worker1\sPromiser\sAPI:\swhen\smultiple\sdb\sconnections\sare\sactive\sthen\suse\sthe\srequested\sconnection\sinstead\sof\salways\sthe\sfirst-opened\sconnection.\sBug\sreported\sin\s[forum:894c330e7f23b177|forum\spost\s894c330e7f23b177]. -D 2023-12-12T17:52:27.818 +C Fix\sthe\sJSON\sobject\slabel\scomparison\sobject\sso\sthat\sit\sworks\scorrectly\seven\nif\sthe\slabel\sends\swith\sescaped\swhitespace. +D 2023-12-12T18:38:53.139 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c cd270629b4b99d065522b45c3d2b5185491b1ab3544843c6f780aa50755d54dc +F src/json.c 59f357a5f88f3a944b97b87217c21727581a40db7273c266c1c378bcc741200e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 840efb33a3934a9bdccad65ea15c70d5c335a517502c9844aaad8405d16d1604 -R 5c3d83f631ee20b7dd8bc56f0a1d21a8 -U stephan -Z 26c52cb9495b2f9f42b5f7d1f0a8e853 +P 194276e18e0268829061c09317e7f9f527a703eb45f1755ff1dd30bd99dc1b68 +R 26c549712092afaef6e0f29bf123d45a +U drh +Z cc3e1f772eba8a30a8b1158560fbf9b2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 481d66b8fb..a9b489978d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -194276e18e0268829061c09317e7f9f527a703eb45f1755ff1dd30bd99dc1b68 \ No newline at end of file +4d5353cadd7b7c5f105bc197f3ec739e2d041472d6b3e939654c9f9cfc2749ae \ No newline at end of file diff --git a/src/json.c b/src/json.c index e94cac6d52..a24a778943 100644 --- a/src/json.c +++ b/src/json.c @@ -2478,8 +2478,10 @@ static SQLITE_NOINLINE int jsonLabelCompareEscaped( ){ u32 cLeft, cRight; assert( rawLeft==0 || rawRight==0 ); - while( nLeft>0 && nRight>0 ){ - if( rawLeft || zLeft[0]!='\\' ){ + while( 1 /*exit-by-return*/ ){ + if( nLeft==0 ){ + cLeft = 0; + }else if( rawLeft || zLeft[0]!='\\' ){ cLeft = ((u8*)zLeft)[0]; zLeft++; nLeft--; @@ -2489,7 +2491,9 @@ static SQLITE_NOINLINE int jsonLabelCompareEscaped( assert( n<=nLeft ); nLeft -= n; } - if( rawRight || zRight[0]!='\\' ){ + if( nRight==0 ){ + cRight = 0; + }else if( rawRight || zRight[0]!='\\' ){ cRight = ((u8*)zRight)[0]; zRight++; nRight--; @@ -2500,8 +2504,8 @@ static SQLITE_NOINLINE int jsonLabelCompareEscaped( nRight -= n; } if( cLeft!=cRight ) return 0; + if( cLeft==0 ) return 1; } - return nLeft==0 && nRight==0; } /* From 001d1e795cfb58c2e0f217374efe2fdef797ad22 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 13 Dec 2023 14:31:15 +0000 Subject: [PATCH 26/80] Improvements to UTF8 handling, and especially the handling of invalid UTF8, in the JSON routines. FossilOrigin-Name: 1b229c1101d6c384a30f343c5e47b471ab084b2d8e81170eb8f642afc1c67e3b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/json.c | 30 ++++++++++++++++++++++++------ src/sqliteInt.h | 1 + src/utf.c | 33 ++++++++++++++++++++++++++++++++- 5 files changed, 66 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index ca6d324ceb..3b83a4c22b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sJSON\sobject\slabel\scomparison\sobject\sso\sthat\sit\sworks\scorrectly\seven\nif\sthe\slabel\sends\swith\sescaped\swhitespace. -D 2023-12-12T18:38:53.139 +C Improvements\sto\sUTF8\shandling,\sand\sespecially\sthe\shandling\sof\sinvalid\sUTF8,\nin\sthe\sJSON\sroutines. +D 2023-12-13T14:31:15.535 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 59f357a5f88f3a944b97b87217c21727581a40db7273c266c1c378bcc741200e +F src/json.c f5b41d1515ea91a43cf55cc3096bd671d06fde2d9d495008787aae114d41b6fe F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -741,7 +741,7 @@ F src/shell.c.in 0cd2ef4b3c814dded5436625ab664d9a973cbc4266a1768e3aa4cbf11bb41ec F src/sqlite.h.in adcc7dbfeea1e69d6d487139a7e90db8a48fe998f3f5bb0f85c683e6a6fa68ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 615524fff083abf9353aa4a3a8adca2d32b3e61e014e62b1a5335c454a386783 +F src/sqliteInt.h 5456977f4449aeb57d837229730edca0aa43059d93093fbeda48e11cffd065f8 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -803,7 +803,7 @@ F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70e F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 -F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 +F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c b22cc9f203a8c0b9ee5338a67f8860347d14845864c10248bebe84518a781677 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c f73bead140670fac1aa4227188827ada52387a5fe0ccff0dd5af2a906754d904 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 194276e18e0268829061c09317e7f9f527a703eb45f1755ff1dd30bd99dc1b68 -R 26c549712092afaef6e0f29bf123d45a +P 4d5353cadd7b7c5f105bc197f3ec739e2d041472d6b3e939654c9f9cfc2749ae +R a4f8b9329a75039f2cc7d3c3b3a11e22 U drh -Z cc3e1f772eba8a30a8b1158560fbf9b2 +Z f5b89926c0a743c6929f7891e073ce93 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a9b489978d..db33165f64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d5353cadd7b7c5f105bc197f3ec739e2d041472d6b3e939654c9f9cfc2749ae \ No newline at end of file +1b229c1101d6c384a30f343c5e47b471ab084b2d8e81170eb8f642afc1c67e3b \ No newline at end of file diff --git a/src/json.c b/src/json.c index a24a778943..f3166187b3 100644 --- a/src/json.c +++ b/src/json.c @@ -2449,8 +2449,8 @@ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ }else if( z[nSkip]=='\\' ){ return nSkip + jsonUnescapeOneChar(&z[nSkip], n-nSkip, piOut); }else{ - *piOut = z[nSkip]; - return nSkip+1; + int sz = sqlite3Utf8ReadLimited((u8*)&z[nSkip], n-nSkip, piOut); + return nSkip + sz; } } default: { @@ -2483,8 +2483,14 @@ static SQLITE_NOINLINE int jsonLabelCompareEscaped( cLeft = 0; }else if( rawLeft || zLeft[0]!='\\' ){ cLeft = ((u8*)zLeft)[0]; - zLeft++; - nLeft--; + if( cLeft>=0xc0 ){ + int sz = sqlite3Utf8ReadLimited((u8*)zLeft, nLeft, &cLeft); + zLeft += sz; + nLeft -= sz; + }else{ + zLeft++; + nLeft--; + } }else{ u32 n = jsonUnescapeOneChar(zLeft, nLeft, &cLeft); zLeft += n; @@ -2495,8 +2501,14 @@ static SQLITE_NOINLINE int jsonLabelCompareEscaped( cRight = 0; }else if( rawRight || zRight[0]!='\\' ){ cRight = ((u8*)zRight)[0]; - zRight++; - nRight--; + if( cRight>=0xc0 ){ + int sz = sqlite3Utf8ReadLimited((u8*)zRight, nRight, &cRight); + zRight += sz; + nRight -= sz; + }else{ + zRight++; + nRight--; + } }else{ u32 n = jsonUnescapeOneChar(zRight, nRight, &cRight); zRight += n; @@ -2916,14 +2928,19 @@ static void jsonReturnFromBlob( u32 szEscape = jsonUnescapeOneChar(&z[iIn], sz-iIn, &v); if( v<=0x7f ){ zOut[iOut++] = (char)v; + }else if( v==0xfffd ){ + /* Silently ignore illegal unicode */ }else if( v<=0x7ff ){ + assert( szEscape>=2 ); zOut[iOut++] = (char)(0xc0 | (v>>6)); zOut[iOut++] = 0x80 | (v&0x3f); }else if( v<0x10000 ){ + assert( szEscape>=3 ); zOut[iOut++] = 0xe0 | (v>>12); zOut[iOut++] = 0x80 | ((v>>6)&0x3f); zOut[iOut++] = 0x80 | (v&0x3f); }else{ + assert( szEscape>=4 ); zOut[iOut++] = 0xf0 | (v>>18); zOut[iOut++] = 0x80 | ((v>>12)&0x3f); zOut[iOut++] = 0x80 | ((v>>6)&0x3f); @@ -2934,6 +2951,7 @@ static void jsonReturnFromBlob( zOut[iOut++] = c; } } /* end for() */ + assert( iOut<=nOut ); zOut[iOut] = 0; sqlite3_result_text(pCtx, zOut, iOut, sqlite3_free); break; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 83226b5750..7d6596909c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -5171,6 +5171,7 @@ int sqlite3Utf16ByteLen(const void *pData, int nChar); #endif int sqlite3Utf8CharLen(const char *pData, int nByte); u32 sqlite3Utf8Read(const u8**); +int sqlite3Utf8ReadLimited(const u8*, int, u32*); LogEst sqlite3LogEst(u64); LogEst sqlite3LogEstAdd(LogEst,LogEst); LogEst sqlite3LogEstFromDouble(double); diff --git a/src/utf.c b/src/utf.c index 5f27babdfc..216864f5c7 100644 --- a/src/utf.c +++ b/src/utf.c @@ -164,7 +164,38 @@ u32 sqlite3Utf8Read( return c; } - +/* +** Read a single UTF8 character out of buffer z[], but reading no +** more than n characters from the buffer. z[] is not zero-terminated. +** +** Return the number of bytes used to construct the character. +** +** Invalid UTF8 might generate a strange result. No effort is made +** to detect invalid UTF8. +** +** At most 4 bytes will be read out of z[]. The return value will always +** be between 1 and 4. +*/ +int sqlite3Utf8ReadLimited( + const u8 *z, + int n, + u32 *piOut +){ + u32 c; + int i = 1; + assert( n>0 ); + c = z[0]; + if( c>=0xc0 ){ + c = sqlite3Utf8Trans1[c-0xc0]; + if( n>4 ) n = 4; + while( i Date: Wed, 13 Dec 2023 15:27:03 +0000 Subject: [PATCH 27/80] Bug fix in the JSONB validator. dbsqlfuzz ac6fa521a08609a642198e7decf64180e750b3c4 FossilOrigin-Name: 3e940a6a08b0a0434650cd3d8dd286e09ad8ab805b0a4d515e57bba5d3608577 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3b83a4c22b..5c94bfcf97 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sUTF8\shandling,\sand\sespecially\sthe\shandling\sof\sinvalid\sUTF8,\nin\sthe\sJSON\sroutines. -D 2023-12-13T14:31:15.535 +C Bug\sfix\sin\sthe\sJSONB\svalidator.\ndbsqlfuzz\sac6fa521a08609a642198e7decf64180e750b3c4 +D 2023-12-13T15:27:03.130 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c f5b41d1515ea91a43cf55cc3096bd671d06fde2d9d495008787aae114d41b6fe +F src/json.c 64e5ed9b56127f6e74db01b0943ebc04af5e4690a6b30937f420d6a7278c57de F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d5353cadd7b7c5f105bc197f3ec739e2d041472d6b3e939654c9f9cfc2749ae -R a4f8b9329a75039f2cc7d3c3b3a11e22 +P 1b229c1101d6c384a30f343c5e47b471ab084b2d8e81170eb8f642afc1c67e3b +R 9712267960fcb4221f5d0c5eee069b9a U drh -Z f5b89926c0a743c6929f7891e073ce93 +Z 12d7bcafc8f212e81442662dadd8f2af # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index db33165f64..4e7898e605 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b229c1101d6c384a30f343c5e47b471ab084b2d8e81170eb8f642afc1c67e3b \ No newline at end of file +3e940a6a08b0a0434650cd3d8dd286e09ad8ab805b0a4d515e57bba5d3608577 \ No newline at end of file diff --git a/src/json.c b/src/json.c index f3166187b3..c484f79564 100644 --- a/src/json.c +++ b/src/json.c @@ -1364,7 +1364,6 @@ static u32 jsonbValidityCheck( if( !jsonIsOk[z[j]] && z[j]!='\'' ){ if( z[j]=='"' ){ if( x==JSONB_TEXTJ ) return j+1; - j++; }else if( z[j]!='\\' || j+1>=k ){ return j+1; }else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){ From 3e4195c60db744c69bb24e71236631a80f2ef547 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 13 Dec 2023 16:45:18 +0000 Subject: [PATCH 28/80] Avoid invoking sqlite3ExprColUsage() on an unresolve column reference. dbsqlfuzz fc34aa62df4de103705d11b807074687ffafbda5. FossilOrigin-Name: ac9314c0e335694b48c613145f5397247bb88c51806cd0dc3ed4ec306db4bbad --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/resolve.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5c94bfcf97..01a664e460 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bug\sfix\sin\sthe\sJSONB\svalidator.\ndbsqlfuzz\sac6fa521a08609a642198e7decf64180e750b3c4 -D 2023-12-13T15:27:03.130 +C Avoid\sinvoking\ssqlite3ExprColUsage()\son\san\sunresolve\scolumn\sreference.\ndbsqlfuzz\sfc34aa62df4de103705d11b807074687ffafbda5. +D 2023-12-13T16:45:18.334 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -734,7 +734,7 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916 +F src/resolve.c da0a596a645cd7a8a9fd030e5126600b5639cc63f8ead18c1b67ff5a8a9b6a7f F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445 F src/shell.c.in 0cd2ef4b3c814dded5436625ab664d9a973cbc4266a1768e3aa4cbf11bb41eca @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1b229c1101d6c384a30f343c5e47b471ab084b2d8e81170eb8f642afc1c67e3b -R 9712267960fcb4221f5d0c5eee069b9a +P 3e940a6a08b0a0434650cd3d8dd286e09ad8ab805b0a4d515e57bba5d3608577 +R 12c352a17a15c54a5cfe08ba07f54fc8 U drh -Z 12d7bcafc8f212e81442662dadd8f2af +Z 9d3792e688e126d78c7c047eb33fe2f8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4e7898e605..3cebbeb2a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e940a6a08b0a0434650cd3d8dd286e09ad8ab805b0a4d515e57bba5d3608577 \ No newline at end of file +ac9314c0e335694b48c613145f5397247bb88c51806cd0dc3ed4ec306db4bbad \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 5f675c1d2d..43592f8abc 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -182,6 +182,7 @@ Bitmask sqlite3ExprColUsed(Expr *pExpr){ assert( ExprUseYTab(pExpr) ); pExTab = pExpr->y.pTab; assert( pExTab!=0 ); + assert( n < pExTab->nCol ); if( (pExTab->tabFlags & TF_HasGenerated)!=0 && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 ){ @@ -784,7 +785,7 @@ static int lookupName( ** If a generated column is referenced, set bits for every column ** of the table. */ - if( pExpr->iColumn>=0 && pMatch!=0 ){ + if( pExpr->iColumn>=0 && cnt==1 && pMatch!=0 ){ pMatch->colUsed |= sqlite3ExprColUsed(pExpr); } From 9e488a5a408099361e3bdd9841c1000945ab43b9 Mon Sep 17 00:00:00 2001 From: larrybr Date: Wed, 13 Dec 2023 20:37:20 +0000 Subject: [PATCH 29/80] In CLI, fix .read inability to open 2GB+ files on WIN32. FossilOrigin-Name: 56c80a62d2e033d64ba5d545ae9cbe3ed7c9d046c0a3fafb6cfa2f0b562d1ef0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 01a664e460..680222b79e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sinvoking\ssqlite3ExprColUsage()\son\san\sunresolve\scolumn\sreference.\ndbsqlfuzz\sfc34aa62df4de103705d11b807074687ffafbda5. -D 2023-12-13T16:45:18.334 +C In\sCLI,\sfix\s.read\sinability\sto\sopen\s2GB+\sfiles\son\sWIN32. +D 2023-12-13T20:37:20.633 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -737,7 +737,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c da0a596a645cd7a8a9fd030e5126600b5639cc63f8ead18c1b67ff5a8a9b6a7f F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445 -F src/shell.c.in 0cd2ef4b3c814dded5436625ab664d9a973cbc4266a1768e3aa4cbf11bb41eca +F src/shell.c.in 0df801a0445af3fc55c7330bcd8396ca5433154bb0f728edc8be2d6f27764361 F src/sqlite.h.in adcc7dbfeea1e69d6d487139a7e90db8a48fe998f3f5bb0f85c683e6a6fa68ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3e940a6a08b0a0434650cd3d8dd286e09ad8ab805b0a4d515e57bba5d3608577 -R 12c352a17a15c54a5cfe08ba07f54fc8 -U drh -Z 9d3792e688e126d78c7c047eb33fe2f8 +P ac9314c0e335694b48c613145f5397247bb88c51806cd0dc3ed4ec306db4bbad +R 44341b29647c99b940d4ac6d2d84c7d7 +U larrybr +Z 039b405ee3a739c6a638ec1362b08fe4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3cebbeb2a1..854ff05586 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac9314c0e335694b48c613145f5397247bb88c51806cd0dc3ed4ec306db4bbad \ No newline at end of file +56c80a62d2e033d64ba5d545ae9cbe3ed7c9d046c0a3fafb6cfa2f0b562d1ef0 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 35ed0cb5c4..26b44e05a9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -736,14 +736,14 @@ static int strlenChar(const char *z){ */ static FILE * openChrSource(const char *zFile){ #if defined(_WIN32) || defined(WIN32) - struct _stat x = {0}; + struct __stat64 x = {0}; # define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) /* On Windows, open first, then check the stream nature. This order ** is necessary because _stat() and sibs, when checking a named pipe, ** effectively break the pipe as its supplier sees it. */ FILE *rv = fopen(zFile, "rb"); if( rv==0 ) return 0; - if( _fstat(_fileno(rv), &x) != 0 + if( _fstat64(_fileno(rv), &x) != 0 || !STAT_CHR_SRC(x.st_mode)){ fclose(rv); rv = 0; From 07117f8118f0f7858126c36ee4d202196f541330 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 14 Dec 2023 13:58:50 +0000 Subject: [PATCH 30/80] Pass subtype information through the aggregate ORDER BY sorter for aggregate functions that use subtype information. FossilOrigin-Name: 3536f4030eab6d650b7ed729d2f71eb6cc3b5fbe16b4e96b99008d66522aaccb --- manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/expr.c | 2 ++ src/select.c | 34 ++++++++++++++++++++++++++++++---- src/sqliteInt.h | 1 + src/vdbe.c | 36 ++++++++++++++++++++++++++++++++++++ test/aggorderby.test | 40 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 123 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 01a664e460..1c3619979e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sinvoking\ssqlite3ExprColUsage()\son\san\sunresolve\scolumn\sreference.\ndbsqlfuzz\sfc34aa62df4de103705d11b807074687ffafbda5. -D 2023-12-13T16:45:18.334 +C Pass\ssubtype\sinformation\sthrough\sthe\saggregate\sORDER\sBY\ssorter\sfor\naggregate\sfunctions\sthat\suse\ssubtype\sinformation. +D 2023-12-14T13:58:50.159 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -686,7 +686,7 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c 05278def9c186b5875d6903ea26148c7461b9ce0344f0fd7be9a0dfea0a4538a +F src/expr.c 8f8ed8a5977a54c678755e43aabb3bbe29984025d086128a93e52675987c34eb F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 @@ -736,12 +736,12 @@ F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c da0a596a645cd7a8a9fd030e5126600b5639cc63f8ead18c1b67ff5a8a9b6a7f F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445 +F src/select.c f69e8e37c8bbe3bb02c890ad38a43c36757b87c7863d071d8ecd720732ed46cb F src/shell.c.in 0cd2ef4b3c814dded5436625ab664d9a973cbc4266a1768e3aa4cbf11bb41eca F src/sqlite.h.in adcc7dbfeea1e69d6d487139a7e90db8a48fe998f3f5bb0f85c683e6a6fa68ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 5456977f4449aeb57d837229730edca0aa43059d93093fbeda48e11cffd065f8 +F src/sqliteInt.h 134457f62bb1d0ff1dd037cc23dd46b1d16efbbfc2211dc2b15c380af731d9ac F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -806,7 +806,7 @@ F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c b22cc9f203a8c0b9ee5338a67f8860347d14845864c10248bebe84518a781677 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c f73bead140670fac1aa4227188827ada52387a5fe0ccff0dd5af2a906754d904 +F src/vdbe.c 96ac876e57f480bd35ec8d74ed992bca6ae9deebe8b527a3a718e7b4714d6c2e F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b @@ -832,7 +832,7 @@ F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggfault.test 777f269d0da5b0c2524c7ff6d99ae9a93db4f1b1839a914dd2a12e3035c29829 F test/aggnested.test ce85a6af7d59c3109e35c5f03b2cd11da1a9b1417371e2f942102d0f0d77fd62 -F test/aggorderby.test e6b98dbbf3ababa96892435d387de2dcf602ef02c2b848d2d817473066f154ba +F test/aggorderby.test cc3abf5de64d46ff66395ca8c2346b66c2576d5aedb7bffc5b0742508856e3bf F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 2ecb8bbd52416642e41c9081182a8df05d42c75637afd4488aace78cc4b69e13 F test/alter.test 3c00eff1e2036b9f93e9cd0f3d3e63750ac87ecb5bc71b9d7bd07cbf2ac4c494 @@ -2153,8 +2153,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3e940a6a08b0a0434650cd3d8dd286e09ad8ab805b0a4d515e57bba5d3608577 -R 12c352a17a15c54a5cfe08ba07f54fc8 +P ac9314c0e335694b48c613145f5397247bb88c51806cd0dc3ed4ec306db4bbad +R 58bb3c6218fca3f2f88129174cf3abf4 +T *branch * agg-orderby-subtype +T *sym-agg-orderby-subtype * +T -sym-trunk * U drh -Z 9d3792e688e126d78c7c047eb33fe2f8 +Z a9d93266587a253c256405246043cc2e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3cebbeb2a1..edaca47980 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac9314c0e335694b48c613145f5397247bb88c51806cd0dc3ed4ec306db4bbad \ No newline at end of file +3536f4030eab6d650b7ed729d2f71eb6cc3b5fbe16b4e96b99008d66522aaccb \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index b5e903d456..756fcc5699 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6821,6 +6821,8 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ }else{ pItem->bOBPayload = 1; } + pItem->bUseSubtype = + (pItem->pFunc->funcFlags & SQLITE_SUBTYPE)!=0; }else{ pItem->iOBTab = -1; } diff --git a/src/select.c b/src/select.c index 1beaf5307d..d5b94e2da2 100644 --- a/src/select.c +++ b/src/select.c @@ -6659,6 +6659,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ assert( pFunc->pFExpr->pLeft!=0 ); assert( pFunc->pFExpr->pLeft->op==TK_ORDER ); assert( ExprUseXList(pFunc->pFExpr->pLeft) ); + assert( pFunc->pFunc!=0 ); pOBList = pFunc->pFExpr->pLeft->x.pList; if( !pFunc->bOBUnique ){ nExtra++; /* One extra column for the OP_Sequence */ @@ -6668,6 +6669,9 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ assert( ExprUseXList(pFunc->pFExpr) ); nExtra += pFunc->pFExpr->x.pList->nExpr; } + if( pFunc->bUseSubtype ){ + nExtra += pFunc->pFExpr->x.pList->nExpr; + } pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOBList, 0, nExtra); if( !pFunc->bOBUnique && pParse->nErr==0 ){ pKeyInfo->nKeyField++; @@ -6694,16 +6698,17 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; if( pF->iOBTab>=0 ){ - /* For an ORDER BY aggregate, calls to OP_AggStep where deferred and - ** all content was stored in emphermal table pF->iOBTab. Extract that - ** content now (in ORDER BY order) and make all calls to OP_AggStep + /* For an ORDER BY aggregate, calls to OP_AggStep were deferred. Inputs + ** were stored in emphermal table pF->iOBTab. Here, we extract those + ** inputs (in ORDER BY order) and make all calls to OP_AggStep ** before doing the OP_AggFinal call. */ int iTop; /* Start of loop for extracting columns */ int nArg; /* Number of columns to extract */ int nKey; /* Key columns to be skipped */ int regAgg; /* Extract into this array */ int j; /* Loop counter */ - + + assert( pF->pFunc!=0 ); nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); @@ -6720,6 +6725,15 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ for(j=nArg-1; j>=0; j--){ sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, nKey+j, regAgg+j); } + if( pF->bUseSubtype ){ + int regSubtype = sqlite3GetTempReg(pParse); + int iBaseCol = nKey + nArg + (pF->bOBPayload==0 && pF->bOBUnique==0); + for(j=nArg-1; j>=0; j--){ + sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, iBaseCol+j, regSubtype); + sqlite3VdbeAddOp2(v, OP_SetSubtype, regSubtype, regAgg+j); + } + sqlite3ReleaseTempReg(pParse, regSubtype); + } sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); @@ -6774,6 +6788,7 @@ static void updateAccumulator( ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); assert( !IsWindowFunc(pF->pFExpr) ); + assert( pF->pFunc!=0 ); pList = pF->pFExpr->x.pList; if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){ Expr *pFilter = pF->pFExpr->y.pWin->pFilter; @@ -6818,6 +6833,9 @@ static void updateAccumulator( if( pF->bOBPayload ){ regAggSz += nArg; } + if( pF->bUseSubtype ){ + regAggSz += nArg; + } regAggSz++; /* One extra register to hold result of MakeRecord */ regAgg = sqlite3GetTempRange(pParse, regAggSz); regDistinct = regAgg; @@ -6830,6 +6848,14 @@ static void updateAccumulator( if( pF->bOBPayload ){ regDistinct = regAgg+jj; sqlite3ExprCodeExprList(pParse, pList, regDistinct, 0, SQLITE_ECEL_DUP); + jj += nArg; + } + if( pF->bUseSubtype ){ + int kk; + int regBase = pF->bOBPayload ? regDistinct : regAgg; + for(kk=0; kknExpr; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7d6596909c..a21425a144 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2866,6 +2866,7 @@ struct AggInfo { int iOBTab; /* Ephemeral table to implement ORDER BY */ u8 bOBPayload; /* iOBTab has payload columns separate from key */ u8 bOBUnique; /* Enforce uniqueness on iOBTab keys */ + u8 bUseSubtype; /* Transfer subtype info through sorter */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ diff --git a/src/vdbe.c b/src/vdbe.c index 4f710d9a20..6d45bbbbbd 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -8674,6 +8674,42 @@ case OP_ClrSubtype: { /* in1 */ break; } +/* Opcode: GetSubtype P1 P2 * * * +** Synopsis: r[P2] = r[P1].subtype +** +** Extract the subtype value from register P1 and write that subtype +** into register P2. If P1 has no subtype, then P1 gets a NULL. +*/ +case OP_GetSubtype: { /* in1 out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Subtype ){ + sqlite3VdbeMemSetInt64(pOut, pIn1->eSubtype); + }else{ + sqlite3VdbeMemSetNull(pOut); + } + break; +} + +/* Opcode: SetSubtype P1 P2 * * * +** Synopsis: r[P2].subtype = r[P1] +** +** Set the subtype value of register P2 to the integer from register P1. +** If P1 is NULL, clear the subtype from p2. +*/ +case OP_SetSubtype: { /* in1 out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Null ){ + pOut->flags &= ~MEM_Subtype; + }else{ + assert( pIn1->flags & MEM_Int ); + pOut->flags |= MEM_Subtype; + pOut->eSubtype = (u8)(pIn1->u.i & 0xff); + } + break; +} + /* Opcode: FilterAdd P1 * P3 P4 * ** Synopsis: filter(P1) += key(P3@P4) ** diff --git a/test/aggorderby.test b/test/aggorderby.test index f94c5ec898..eed1f83a7e 100644 --- a/test/aggorderby.test +++ b/test/aggorderby.test @@ -118,5 +118,45 @@ do_execsql_test aggorderby-8.2 { SELECT sum(DISTINCT x ORDER BY y) FROM c; } 6 +# Subtype information is transfered through the sorter for aggregates +# that make use of subtype info. +# +do_execsql_test aggorderby-9.0 { + WITH c(x,y) AS (VALUES + ('{a:3}', 3), + ('[1,1]', 1), + ('[4,4]', 4), + ('{x:2}', 2)) + SELECT json_group_array(json(x) ORDER BY y) FROM c; +} {{[[1,1],{"x":2},{"a":3},[4,4]]}} +do_execsql_test aggorderby-9.1 { + WITH c(x,y) AS (VALUES + ('[4,4]', 4), + ('{a:3}', 3), + ('[4,4]', 4), + ('[1,1]', 1), + ('[4,4]', 4), + ('{x:2}', 2)) + SELECT json_group_array(DISTINCT json(x) ORDER BY y) FROM c; +} {{[[1,1],{"x":2},{"a":3},[4,4]]}} +do_execsql_test aggorderby-9.2 { + WITH c(x,y) AS (VALUES + ('{a:3}', 3), + ('[1,1]', 1), + ('[4,4]', 4), + ('{x:2}', 2)) + SELECT json_group_array(json(x) ORDER BY json(x)) FROM c; +} {{[[1,1],[4,4],{"a":3},{"x":2}]}} +do_execsql_test aggorderby-9.3 { + WITH c(x,y) AS (VALUES + ('[4,4]', 4), + ('{a:3}', 3), + ('[4,4]', 4), + ('[1,1]', 1), + ('[4,4]', 4), + ('{x:2}', 2)) + SELECT json_group_array(DISTINCT json(x) ORDER BY json(x)) FROM c; +} {{[[1,1],[4,4],{"a":3},{"x":2}]}} + finish_test From e6289d6dd041d7b908a8c4b936fe95425f51f5ce Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Dec 2023 15:31:35 +0000 Subject: [PATCH 31/80] Improve the error message returned by an fts5 'rebuild' command on an external content table if there is a problem with the content table or view. FossilOrigin-Name: 0fbf4b8a58fde1c187908934da6f59999b146f32e07ac255cc531c5c4d7007fd --- ext/fts5/fts5_storage.c | 2 +- ext/fts5/test/fts5content.test | 35 ++++++++++++++++++++++++++++++++++ manifest | 17 ++++++++--------- manifest.uuid | 2 +- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 9480da7c52..a04b152fb0 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -673,7 +673,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ } if( rc==SQLITE_OK ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg); } while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ diff --git a/ext/fts5/test/fts5content.test b/ext/fts5/test/fts5content.test index ca2726a902..986d7f3311 100644 --- a/ext/fts5/test/fts5content.test +++ b/ext/fts5/test/fts5content.test @@ -293,4 +293,39 @@ do_catchsql_test 7.2.5 { SELECT * FROM t1('abc') ORDER BY rank; } {1 {recursively defined fts5 content table}} +#--------------------------------------------------------------------------- +# Check that if the content table is a view, and that view contains an +# error, a reasonable error message is returned if the user tries to +# read from the view via the fts5 table. +# +reset_db +do_execsql_test 8.1 { + CREATE VIEW a1 AS + SELECT 1 AS r, text_value(1) AS t + UNION ALL + SELECT 2 AS r, text_value(2) AS t; + + CREATE VIRTUAL TABLE t1 USING fts5(t, content='a1', content_rowid='r'); +} + +foreach {tn sql} { + 1 "SELECT * FROM t1" + 2 "INSERT INTO t1(t1) VALUES('rebuild')" + 3 "SELECT * FROM t1 WHERE rowid=1" +} { + do_catchsql_test 8.2.$tn $sql {1 {no such function: text_value}} +} + +proc text_value {i} { + if {$i==1} { return "one" } + if {$i==2} { return "two" } + return "many" +} +db func text_value text_value + +do_execsql_test 8.3.1 { SELECT * FROM t1 } {one two} +do_execsql_test 8.3.2 { INSERT INTO t1(t1) VALUES('rebuild') } +do_execsql_test 8.3.3 { SELECT * FROM t1 WHERE rowid=1 } {one} +do_execsql_test 8.3.4 { SELECT rowid FROM t1('two') } {2} + finish_test diff --git a/manifest b/manifest index e1e828f898..48d278ebba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\saggregate\sorder-by\sso\sthat\sit\stransmits\ssubtype\sinformation\sthrough\sthe\nsorter.\s\sFix\sfor\sthe\sdeficiency\sreported\sby\n[forum:/forumpost/87347ad2fb5a8f76|forum\spost\s87347ad2fb5a8f76]. -D 2023-12-14T15:11:39.232 +C Improve\sthe\serror\smessage\sreturned\sby\san\sfts5\s'rebuild'\scommand\son\san\sexternal\scontent\stable\sif\sthere\sis\sa\sproblem\swith\sthe\scontent\stable\sor\sview. +D 2023-12-14T15:31:35.537 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -98,7 +98,7 @@ F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d1 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c ed206045ff0f2226d870fa41fba45f738c0cc953ab74ba68477091b9a574ccd3 F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933 -F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d +F ext/fts5/fts5_storage.c f9e31b0d155e9b2c92d5d3a09ad7a56b937fbf1c7f962e10f4ca6281349f3934 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -135,7 +135,7 @@ F ext/fts5/test/fts5columnsize.test 45459ce4dd9fd853b6044cdc9674921bff89e3d840f3 F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825ce437d26afe0817f F ext/fts5/test/fts5conflict.test bf6030a77dbb1bedfcc42e589ed7980846c995765d77460551e448b56d741244 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4 -F ext/fts5/test/fts5content.test 219a4e49386b9b197b9b7cadca97ea10ddff858ecd8b763a1cb8bb07575afc2a +F ext/fts5/test/fts5content.test 282b373c58c8e798568ec2ced18b23f29bffa8d61317a0e51a035000ad6cd731 F ext/fts5/test/fts5contentless.test 1cd1237894eeff11feb1ff8180044eac0b17dde22c181f7a722f2dcbfdb3377c F ext/fts5/test/fts5contentless2.test 14c83bdacf8230f5f7ca74ecf2926b87d8a7cb788a69ce9937020428ac4fe192 F ext/fts5/test/fts5contentless3.test 353d871c5ea08992aed3e2ebda0b1bdc35116cd24fe330fe7cf05be1e2b49fd7 @@ -2153,9 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 56c80a62d2e033d64ba5d545ae9cbe3ed7c9d046c0a3fafb6cfa2f0b562d1ef0 3536f4030eab6d650b7ed729d2f71eb6cc3b5fbe16b4e96b99008d66522aaccb -R a357e72346e8e28e4704317786acd8b6 -T +closed 3536f4030eab6d650b7ed729d2f71eb6cc3b5fbe16b4e96b99008d66522aaccb -U drh -Z 5db0e9f6ed931739d68be6389c9135e2 +P d302a389460d0c15775a8b5f5afbac2c1d8a91bc282bc9b04c583ca04a8c09c6 +R f189902254bb18753480255d64c90698 +U dan +Z a26bc23a688b8f3ee065a8250ea163d3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6404cfd64c..f7cbd2763f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d302a389460d0c15775a8b5f5afbac2c1d8a91bc282bc9b04c583ca04a8c09c6 \ No newline at end of file +0fbf4b8a58fde1c187908934da6f59999b146f32e07ac255cc531c5c4d7007fd \ No newline at end of file From 4a5c96ace3e8d5cee0c68106d6442499627a58dd Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 14 Dec 2023 15:38:57 +0000 Subject: [PATCH 32/80] Fix harmless compiler warnings in JSON and FTS5. FossilOrigin-Name: 90135efccfeb1046f002bfcbd8dfec9a1a3b40cbe1b5c714ae065b06368e354f --- ext/fts5/fts5_expr.c | 7 ++++--- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/json.c | 6 +++--- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index bc7c9741ee..2f2bf28260 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1740,7 +1740,9 @@ static int fts5ParseTokenize( memset(pSyn, 0, (size_t)nByte); pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); pSyn->nFullTerm = pSyn->nQueryTerm = nToken; - if( pCtx->pConfig->bTokendata ) pSyn->nQueryTerm = strlen(pSyn->pTerm); + if( pCtx->pConfig->bTokendata ){ + pSyn->nQueryTerm = (int)strlen(pSyn->pTerm); + } memcpy(pSyn->pTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; @@ -1769,7 +1771,7 @@ static int fts5ParseTokenize( pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); pTerm->nFullTerm = pTerm->nQueryTerm = nToken; if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ - pTerm->nQueryTerm = strlen(pTerm->pTerm); + pTerm->nQueryTerm = (int)strlen(pTerm->pTerm); } } } @@ -3247,4 +3249,3 @@ void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){ } } } - diff --git a/manifest b/manifest index 48d278ebba..af620e1aa0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\serror\smessage\sreturned\sby\san\sfts5\s'rebuild'\scommand\son\san\sexternal\scontent\stable\sif\sthere\sis\sa\sproblem\swith\sthe\scontent\stable\sor\sview. -D 2023-12-14T15:31:35.537 +C Fix\sharmless\scompiler\swarnings\sin\sJSON\sand\sFTS5. +D 2023-12-14T15:38:57.969 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -94,7 +94,7 @@ F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e +F ext/fts5/fts5_expr.c 248ecadbacdbeb85c433f907e57bb91224678c829e57ecf098e3543b5df8c3f9 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c ed206045ff0f2226d870fa41fba45f738c0cc953ab74ba68477091b9a574ccd3 F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 64e5ed9b56127f6e74db01b0943ebc04af5e4690a6b30937f420d6a7278c57de +F src/json.c 55aaffca9b1a136ea5937564f2159f300c2d7c3711cdb1c23356a7d27885827b F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d302a389460d0c15775a8b5f5afbac2c1d8a91bc282bc9b04c583ca04a8c09c6 -R f189902254bb18753480255d64c90698 -U dan -Z a26bc23a688b8f3ee065a8250ea163d3 +P 0fbf4b8a58fde1c187908934da6f59999b146f32e07ac255cc531c5c4d7007fd +R 067cf0081390ed7a5ecc529d5227732e +U drh +Z effe63d1d640400eeacf84541f3b1661 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f7cbd2763f..4e993e1093 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0fbf4b8a58fde1c187908934da6f59999b146f32e07ac255cc531c5c4d7007fd \ No newline at end of file +90135efccfeb1046f002bfcbd8dfec9a1a3b40cbe1b5c714ae065b06368e354f \ No newline at end of file diff --git a/src/json.c b/src/json.c index c484f79564..a1441e8099 100644 --- a/src/json.c +++ b/src/json.c @@ -1535,7 +1535,7 @@ json_parse_restart: break; }else{ if( jsonIsspace(z[j]) ){ - j += 1 + strspn(&z[j+1], jsonSpaces); + j += 1 + (u32)strspn(&z[j+1], jsonSpaces); if( z[j]==',' ){ continue; }else if( z[j]=='}' ){ @@ -1587,7 +1587,7 @@ json_parse_restart: break; }else{ if( jsonIsspace(z[j]) ){ - j += 1 + strspn(&z[j+1], jsonSpaces); + j += 1 + (u32)strspn(&z[j+1], jsonSpaces); if( z[j]==',' ){ continue; }else if( z[j]==']' ){ @@ -1850,7 +1850,7 @@ json_parse_restart: case 0x0a: case 0x0d: case 0x20: { - i += 1 + strspn(&z[i+1], jsonSpaces); + i += 1 + (u32)strspn(&z[i+1], jsonSpaces); goto json_parse_restart; } case 0x0b: From eb78d56ec50813b03009d2422499dbb4c8b64726 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 14 Dec 2023 16:34:47 +0000 Subject: [PATCH 33/80] Add assert()s to FTS5 to fix static analyzer warnings. FossilOrigin-Name: 27d4a89a5ff96b7b7fc5dc9650e1269f7c7edf91de9b9aafce40be9ecc8b95e9 --- ext/fts5/fts5_buffer.c | 3 +++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index b9614e1290..891ef0203a 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -68,6 +68,7 @@ void sqlite3Fts5BufferAppendBlob( ){ if( nData ){ if( fts5BufferGrow(pRc, pBuf, nData) ) return; + assert( pBuf->p!=0 ); memcpy(&pBuf->p[pBuf->n], pData, nData); pBuf->n += nData; } @@ -169,6 +170,7 @@ int sqlite3Fts5PoslistNext64( i64 *piOff /* IN/OUT: Current offset */ ){ int i = *pi; + assert( a!=0 || i==0 ); if( i>=n ){ /* EOF */ *piOff = -1; @@ -176,6 +178,7 @@ int sqlite3Fts5PoslistNext64( }else{ i64 iOff = *piOff; u32 iVal; + assert( a!=0 ); fts5FastGetVarint32(a, i, iVal); if( iVal<=1 ){ if( iVal==0 ){ diff --git a/manifest b/manifest index af620e1aa0..31ed218cdb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sJSON\sand\sFTS5. -D 2023-12-14T15:38:57.969 +C Add\sassert()s\sto\sFTS5\sto\sfix\sstatic\sanalyzer\swarnings. +D 2023-12-14T16:34:47.115 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb0 F ext/fts5/fts5.h 9d7867cc63631296462c90bafe57c4881b56e480fa510ffaee6a77049258c714 F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad -F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 +F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c 248ecadbacdbeb85c433f907e57bb91224678c829e57ecf098e3543b5df8c3f9 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0fbf4b8a58fde1c187908934da6f59999b146f32e07ac255cc531c5c4d7007fd -R 067cf0081390ed7a5ecc529d5227732e +P 90135efccfeb1046f002bfcbd8dfec9a1a3b40cbe1b5c714ae065b06368e354f +R de9e2e3a911515ec6a18e21a77463b2b U drh -Z effe63d1d640400eeacf84541f3b1661 +Z 4f0b96c5c3315c731e117e005cc67ca9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4e993e1093..b69b95b22d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90135efccfeb1046f002bfcbd8dfec9a1a3b40cbe1b5c714ae065b06368e354f \ No newline at end of file +27d4a89a5ff96b7b7fc5dc9650e1269f7c7edf91de9b9aafce40be9ecc8b95e9 \ No newline at end of file From e98238561911f85e54af3e6f26e47c2de8d1cda1 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 14 Dec 2023 22:01:55 +0000 Subject: [PATCH 34/80] Use SQLITE_STRICT_SUBTYPE=1 by default for the JNI and WASM builds unless they're explicitly built with SQLITE_STRICT_SUBTYPE=0. FossilOrigin-Name: 990211357badf0ab08bd34cf6d25b58849d0fd8503e289c1839fc837a74e1909 --- ext/jni/src/c/sqlite3-jni.c | 6 ++++++ ext/wasm/api/sqlite3-wasm.c | 6 ++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index 14c447acd5..5deff19ef1 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -107,6 +107,12 @@ # undef SQLITE_OMIT_UTF16 1 #endif +/**********************************************************************/ +/* SQLITE_S... */ +#ifndef SQLITE_STRICT_SUBTYPE +# define SQLITE_STRICT_SUBTYPE 1 +#endif + /**********************************************************************/ /* SQLITE_T... */ #ifndef SQLITE_TEMP_STORE diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 300307e5ea..3e22c799c4 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -147,6 +147,12 @@ # define SQLITE_OS_KV_OPTIONAL 1 #endif +/**********************************************************************/ +/* SQLITE_S... */ +#ifndef SQLITE_STRICT_SUBTYPE +# define SQLITE_STRICT_SUBTYPE 1 +#endif + /**********************************************************************/ /* SQLITE_T... */ #ifndef SQLITE_TEMP_STORE diff --git a/manifest b/manifest index 31ed218cdb..10f71ea473 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sassert()s\sto\sFTS5\sto\sfix\sstatic\sanalyzer\swarnings. -D 2023-12-14T16:34:47.115 +C Use\sSQLITE_STRICT_SUBTYPE=1\sby\sdefault\sfor\sthe\sJNI\sand\sWASM\sbuilds\sunless\sthey're\sexplicitly\sbuilt\swith\sSQLITE_STRICT_SUBTYPE=0. +D 2023-12-14T22:01:55.094 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -250,7 +250,7 @@ F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a3 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa -F ext/jni/src/c/sqlite3-jni.c 6040c0de97644a1fb14bb589ee9f2f4208f6e6b165d14a0e33ed24945b118838 +F ext/jni/src/c/sqlite3-jni.c c1292e690a20c7787a63e8d8ac6e2dfed49c97282ed056a7cfda5da461f0b7d8 F ext/jni/src/c/sqlite3-jni.h 913ab8e8fee432ae40f0e387c8231118d17053714703f5ded18202912a8a3fbf F ext/jni/src/org/sqlite/jni/annotation/Experimental.java 8603498634e41d0f7c70f661f64e05df64376562ea8f126829fd1e0cdd47e82b F ext/jni/src/org/sqlite/jni/annotation/NotNull.java 38e7e58a69b26dc100e458b31dfa3b2a7d67bc36d051325526ef1987d5bc8a24 @@ -601,7 +601,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 595953994aa3ae2287c889c4da39ab3d6f17b6461ecf4bec334b7a3faafddb02 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91ccfafc38b400fe36e90bdda88e -F ext/wasm/api/sqlite3-wasm.c d0e09eb5ed3743c00294e30019e591c3aa150572ae7ffe8a8994568a7377589f +F ext/wasm/api/sqlite3-wasm.c f280d4ea917d213ae95668dfcd173a2c2ef21a0a4bf9aeb9fcd0edaf1b21ba4b F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js a94e3f0ca25a777bb73779368f97be0e103e02d067ad3ee3e9c4cc5bcefbd01c F ext/wasm/api/sqlite3-worker1.c-pp.js a541112aa51e16705f13a99bb943c64efe178aa28c86704a955f8fd9afe4ba37 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 90135efccfeb1046f002bfcbd8dfec9a1a3b40cbe1b5c714ae065b06368e354f -R de9e2e3a911515ec6a18e21a77463b2b -U drh -Z 4f0b96c5c3315c731e117e005cc67ca9 +P 27d4a89a5ff96b7b7fc5dc9650e1269f7c7edf91de9b9aafce40be9ecc8b95e9 +R b474fa481488202beb159b4a603860df +U stephan +Z 11ee34413d1c8cda564d86731ed3e557 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b69b95b22d..818356391f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27d4a89a5ff96b7b7fc5dc9650e1269f7c7edf91de9b9aafce40be9ecc8b95e9 \ No newline at end of file +990211357badf0ab08bd34cf6d25b58849d0fd8503e289c1839fc837a74e1909 \ No newline at end of file From f062506002c10e2cdc0adc563dc2ce088f5f4614 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 15 Dec 2023 13:38:47 +0000 Subject: [PATCH 35/80] Correct --enable-sab flag in ext/wasm/GNUmakefile to fix a silent alhttpd args-parsing error. FossilOrigin-Name: 7b9b757d872a31395b0f6454e2309a6a4664b8bdd8749f6a15371cbe72c05b60 --- ext/wasm/GNUmakefile | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 3ef478612c..0510af74ab 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -1087,4 +1087,4 @@ endif # Run local web server for the test/demo pages. httpd: - althttpd -max-age 1 -enable-sab -page index.html + althttpd -max-age 1 -enable-sab 1 -page index.html diff --git a/manifest b/manifest index 10f71ea473..6c0a3548cb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sSQLITE_STRICT_SUBTYPE=1\sby\sdefault\sfor\sthe\sJNI\sand\sWASM\sbuilds\sunless\sthey're\sexplicitly\sbuilt\swith\sSQLITE_STRICT_SUBTYPE=0. -D 2023-12-14T22:01:55.094 +C Correct\s--enable-sab\sflag\sin\sext/wasm/GNUmakefile\sto\sfix\sa\ssilent\salhttpd\sargs-parsing\serror. +D 2023-12-15T13:38:47.959 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -574,7 +574,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 57439eec2b8b4d4074e5d861e93aab3f32bb0f44339a2b472c23f4e638b7e8a3 +F ext/wasm/GNUmakefile 62403519b233dbe23e1cac30969714c4043a96c5bc2614e551a07a81c543c493 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27d4a89a5ff96b7b7fc5dc9650e1269f7c7edf91de9b9aafce40be9ecc8b95e9 -R b474fa481488202beb159b4a603860df +P 990211357badf0ab08bd34cf6d25b58849d0fd8503e289c1839fc837a74e1909 +R c42864716310151758eb639de7583cae U stephan -Z 11ee34413d1c8cda564d86731ed3e557 +Z e5ecf8b504178fb4baeba45ef6151951 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 818356391f..b6a7631b4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -990211357badf0ab08bd34cf6d25b58849d0fd8503e289c1839fc837a74e1909 \ No newline at end of file +7b9b757d872a31395b0f6454e2309a6a4664b8bdd8749f6a15371cbe72c05b60 \ No newline at end of file From 76dbb58700ea9410cec67fc017916290bad6dc80 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Dec 2023 14:33:50 +0000 Subject: [PATCH 36/80] Avoid running the "no_mutex_try" tests with SQLITE_ENABLE_SETLK_TIMEOUT builds as part of the release test. FossilOrigin-Name: 6b4e1344a28c213cbe8fb97f7f3f6688de93fb73ed96bf460ff74c959da1a712 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqlite.h.in | 8 +++++--- test/testrunner_data.tcl | 5 ++++- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 6c0a3548cb..365925f812 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\s--enable-sab\sflag\sin\sext/wasm/GNUmakefile\sto\sfix\sa\ssilent\salhttpd\sargs-parsing\serror. -D 2023-12-15T13:38:47.959 +C Avoid\srunning\sthe\s"no_mutex_try"\stests\swith\sSQLITE_ENABLE_SETLK_TIMEOUT\sbuilds\sas\spart\sof\sthe\srelease\stest. +D 2023-12-15T14:33:50.141 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -738,7 +738,7 @@ F src/resolve.c da0a596a645cd7a8a9fd030e5126600b5639cc63f8ead18c1b67ff5a8a9b6a7f F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c f69e8e37c8bbe3bb02c890ad38a43c36757b87c7863d071d8ecd720732ed46cb F src/shell.c.in 0df801a0445af3fc55c7330bcd8396ca5433154bb0f728edc8be2d6f27764361 -F src/sqlite.h.in adcc7dbfeea1e69d6d487139a7e90db8a48fe998f3f5bb0f85c683e6a6fa68ca +F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 F src/sqliteInt.h 134457f62bb1d0ff1dd037cc23dd46b1d16efbbfc2211dc2b15c380af731d9ac @@ -1664,7 +1664,7 @@ F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede F test/testrunner.tcl e18d71f2e797da808ba6d31335e504ed6b2791581b89287a72b697a2f31b1ea1 -F test/testrunner_data.tcl e4d5017290a6d5c11785e36cc94c67d8bb950c8cdc2dbe4c1db2a3a583812560 +F test/testrunner_data.tcl 72bbd60e8ffbe5694cf871cbe8f8f6e542c9f1e6a33765309331aeb5e4f16553 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 990211357badf0ab08bd34cf6d25b58849d0fd8503e289c1839fc837a74e1909 -R c42864716310151758eb639de7583cae -U stephan -Z e5ecf8b504178fb4baeba45ef6151951 +P 7b9b757d872a31395b0f6454e2309a6a4664b8bdd8749f6a15371cbe72c05b60 +R 93bf736458df619f69356c11ae1b79c8 +U dan +Z 5cc92eee4b8244bffbd15f83cef00a45 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b6a7631b4c..b51f281d83 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b9b757d872a31395b0f6454e2309a6a4664b8bdd8749f6a15371cbe72c05b60 \ No newline at end of file +6b4e1344a28c213cbe8fb97f7f3f6688de93fb73ed96bf460ff74c959da1a712 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1f53438142..4a19fe918a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8039,9 +8039,11 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable -** behavior.)^ +** will always return SQLITE_BUSY. In most cases the SQLite core only uses +** sqlite3_mutex_try() as an optimization, so this is acceptable +** behavior. The exceptions are unix builds that set the +** SQLITE_ENABLE_SETLK_TIMEOUT build option. In that case a working +** sqlite3_mutex_try() is required.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was ** previously entered by the same thread. The behavior diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index c4e24c4382..6ca2a80f7a 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -30,7 +30,7 @@ namespace eval trd { set tcltest(osx.Locking-Style) veryquick set tcltest(osx.Have-Not) veryquick - set tcltest(osx.Apple) all + set tcltest(osx.Apple) all_less_no_mutex_try set tcltest(win.Stdcall) veryquick set tcltest(win.Have-Not) veryquick @@ -367,6 +367,9 @@ proc trd_configs {platform bld} { set clist $all_configs } elseif {$clist=="all_plus_autovacuum_crash"} { set clist [concat $all_configs autovacuum_crash] + } elseif {$clist=="all_less_no_mutex_try"} { + set idx [lsearch $all_configs no_mutex_try] + set clist [lreplace $all_configs $idx $idx] } } From f2194605f832a08e6aeccbe0adcc64997bc01754 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Dec 2023 15:17:39 +0000 Subject: [PATCH 37/80] Do not run test script fts5origintest4.test with either "memsubsys1" or "mmap" permutations. FossilOrigin-Name: 05a63d9603ef42cbee6dadff72d97583a9c78e549f70e9a808534d5c1ae7c28a --- ext/fts5/test/fts5origintext4.test | 13 +++++++++++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ext/fts5/test/fts5origintext4.test b/ext/fts5/test/fts5origintext4.test index 8973a24b05..b9a889a514 100644 --- a/ext/fts5/test/fts5origintext4.test +++ b/ext/fts5/test/fts5origintext4.test @@ -21,6 +21,19 @@ ifcapable !fts5 { return } +# The tests below verify that a doclist-index is used to limit the number +# of pages loaded into the cache. It does this by querying sqlite3_db_status() +# for the amount of memory used by the pager cache. +# +# memsubsys1 effectively limits the page-cache to 24 pages. Which masks +# the effect tested by the tests in this file. And "mmap" prevents the +# cache from being used, also preventing these tests from working. +# +if {[permutation]=="memsubsys1" || [permutation]=="mmap"} { + finish_test + return +} + sqlite3_fts5_register_origintext db do_execsql_test 1.0 { CREATE VIRTUAL TABLE ft USING fts5( diff --git a/manifest b/manifest index 365925f812..38b51746de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\srunning\sthe\s"no_mutex_try"\stests\swith\sSQLITE_ENABLE_SETLK_TIMEOUT\sbuilds\sas\spart\sof\sthe\srelease\stest. -D 2023-12-15T14:33:50.141 +C Do\snot\srun\stest\sscript\sfts5origintest4.test\swith\seither\s"memsubsys1"\sor\s"mmap"\spermutations. +D 2023-12-15T15:17:39.739 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -194,7 +194,7 @@ F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842 F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39 F ext/fts5/test/fts5origintext2.test f3b9436de540828d01f0672df855b09ebc0863e126d5b56234701d71dfa73634 F ext/fts5/test/fts5origintext3.test 0d25933506600452a5ab3873cbb418ed5f2de2446c3672b9997b1ea104b0e7f0 -F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871 +F ext/fts5/test/fts5origintext4.test a33e8f64b9762e0e0c722ac2b301017e1d7745635724c1ca04b2c010b451fab4 F ext/fts5/test/fts5origintext5.test a037bdf7235a22033c4663837bdb12d9738245464a3ac2f60c71fc40d07ede7d F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7b9b757d872a31395b0f6454e2309a6a4664b8bdd8749f6a15371cbe72c05b60 -R 93bf736458df619f69356c11ae1b79c8 +P 6b4e1344a28c213cbe8fb97f7f3f6688de93fb73ed96bf460ff74c959da1a712 +R 5009aa3a7688d2e0f2d3fd5b288c1e45 U dan -Z 5cc92eee4b8244bffbd15f83cef00a45 +Z a7e6258d44595ae5e733a4da3cda0926 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b51f281d83..433a91385f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b4e1344a28c213cbe8fb97f7f3f6688de93fb73ed96bf460ff74c959da1a712 \ No newline at end of file +05a63d9603ef42cbee6dadff72d97583a9c78e549f70e9a808534d5c1ae7c28a \ No newline at end of file From baa1aba8d7ebb630de2d26197e86a7338ca49e8c Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 15 Dec 2023 15:22:03 +0000 Subject: [PATCH 38/80] Fix a new JSON test case so that it works even if SQLITE_OMIT_VIRTUALTABLE is defined. FossilOrigin-Name: b995aae510888a9746b46545d176a0885d4738e1f1bc0b7ad7937ed023efd7d6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/json502.test | 10 ++++++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 38b51746de..e32e4e2570 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\srun\stest\sscript\sfts5origintest4.test\swith\seither\s"memsubsys1"\sor\s"mmap"\spermutations. -D 2023-12-15T15:17:39.739 +C Fix\sa\snew\sJSON\stest\scase\sso\sthat\sit\sworks\seven\sif\sSQLITE_OMIT_VIRTUALTABLE\nis\sdefined. +D 2023-12-15T15:22:03.625 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1339,7 +1339,7 @@ F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a888011 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test e64a8d73677fbae67886642cd5076e2ef3efe89f8483b87595cf9c030216c9bd F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330 -F test/json502.test 3c697e506fc38ccb455b49660b21b6e62e08ede0f2d0c869a7d171e17809093c +F test/json502.test 84634d3dbb521d2814e43624025b760c6198456c8197bbec6c977c0236648f5b F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6b4e1344a28c213cbe8fb97f7f3f6688de93fb73ed96bf460ff74c959da1a712 -R 5009aa3a7688d2e0f2d3fd5b288c1e45 -U dan -Z a7e6258d44595ae5e733a4da3cda0926 +P 05a63d9603ef42cbee6dadff72d97583a9c78e549f70e9a808534d5c1ae7c28a +R 3314c5bf0699d50c465bd6395bdb0dc5 +U drh +Z 6217845991b9f421ba875e859fa5fd55 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 433a91385f..a473849eb8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -05a63d9603ef42cbee6dadff72d97583a9c78e549f70e9a808534d5c1ae7c28a \ No newline at end of file +b995aae510888a9746b46545d176a0885d4738e1f1bc0b7ad7937ed023efd7d6 \ No newline at end of file diff --git a/test/json502.test b/test/json502.test index 48c372c4f8..cc14b8cb71 100644 --- a/test/json502.test +++ b/test/json502.test @@ -47,7 +47,7 @@ do_execsql_test 3.2 { db null null do_execsql_test 3.3 { - DROP TABLE t1; + DROP TABLE IF EXISTS t1; CREATE TABLE t1(x); INSERT INTO t1 VALUES(json_insert('{}','$.a\',111,'$."b\\"',222)); INSERT INTO t1 VALUES(jsonb_insert('{}','$.a\',111,'$."b\\"',222)); @@ -58,8 +58,10 @@ do_execsql_test 3.4 { SELECT json_patch('{"a\x62c":123}','{"ab\x63":456}') ->> 'abc'; } 456 -do_execsql_test 4.1 { - SELECT * FROM json_tree('{"\u0017":1}','$."\x17"'); -} {{\x17} 1 integer 1 1 null {$."\x17"} {$}} +ifcapable vtab { + do_execsql_test 4.1 { + SELECT * FROM json_tree('{"\u0017":1}','$."\x17"'); + } {{\x17} 1 integer 1 1 null {$."\x17"} {$}} +} finish_test From b7fad0dca321af3263b9352fd6d49a6d4f7649c2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 15 Dec 2023 16:28:02 +0000 Subject: [PATCH 39/80] Add mention of --buildonly and --dryrun to the testrunner.tcl usage screen. FossilOrigin-Name: 23b92d915c12ee768857e2c3c961832f390cad9b53b8bcfc2b97664baab25bb7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e32e4e2570..39a8074084 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\snew\sJSON\stest\scase\sso\sthat\sit\sworks\seven\sif\sSQLITE_OMIT_VIRTUALTABLE\nis\sdefined. -D 2023-12-15T15:22:03.625 +C Add\smention\sof\s--buildonly\sand\s--dryrun\sto\sthe\stestrunner.tcl\susage\sscreen. +D 2023-12-15T16:28:02.853 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1663,7 +1663,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede -F test/testrunner.tcl e18d71f2e797da808ba6d31335e504ed6b2791581b89287a72b697a2f31b1ea1 +F test/testrunner.tcl 8e2a5c7550b78d3283eee6103104ae2bcf56aa1df892dbd1608f27b93ebf4de8 F test/testrunner_data.tcl 72bbd60e8ffbe5694cf871cbe8f8f6e542c9f1e6a33765309331aeb5e4f16553 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 05a63d9603ef42cbee6dadff72d97583a9c78e549f70e9a808534d5c1ae7c28a -R 3314c5bf0699d50c465bd6395bdb0dc5 +P b995aae510888a9746b46545d176a0885d4738e1f1bc0b7ad7937ed023efd7d6 +R e9d176533dea184874a9bb1447a17bcc U drh -Z 6217845991b9f421ba875e859fa5fd55 +Z 1206a14cd99f29a5172802ae7d99e8c8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a473849eb8..0d75d9cb1e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b995aae510888a9746b46545d176a0885d4738e1f1bc0b7ad7937ed023efd7d6 \ No newline at end of file +23b92d915c12ee768857e2c3c961832f390cad9b53b8bcfc2b97664baab25bb7 \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl index edefdcd156..0c704daf21 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -58,6 +58,8 @@ Usage: $a0 status where SWITCHES are: + --buildonly + --dryrun --jobs NUMBER-OF-JOBS --zipvfs ZIPVFS-SOURCE-DIR From 97f7ead3f301479fdf45981eb6719622daf20998 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Dec 2023 19:26:16 +0000 Subject: [PATCH 40/80] Avoid expiring prepared statements in the middle of an integrity-check. FossilOrigin-Name: 88beb48472da4667c0727c8ebabe046ea526450ff837fe789d041ed3f1ff105e --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/vtab.c | 1 - test/fts3integrity.test | 42 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 test/fts3integrity.test diff --git a/manifest b/manifest index 39a8074084..6a0e11442e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smention\sof\s--buildonly\sand\s--dryrun\sto\sthe\stestrunner.tcl\susage\sscreen. -D 2023-12-15T16:28:02.853 +C Avoid\sexpiring\sprepared\sstatements\sin\sthe\smiddle\sof\san\sintegrity-check. +D 2023-12-15T19:26:16.801 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -816,7 +816,7 @@ F src/vdbemem.c 0012d5f01cc866833847c2f3ae4c318ac53a1cb3d28acad9c35e688039464cf0 F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c 2143db7db0ceed69b21422581f434baffc507a08d831565193a7a02882a1b6a7 -F src/vtab.c 154725ebecd3bc02f7fbd7ad3974334f73fff76e02a964e828e48a7c5fb7efff +F src/vtab.c 11948e105f56e84099ca17f1f434b1944539ea84de26d0d767eadfbc670ce1ea F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 @@ -1169,6 +1169,7 @@ F test/fts3fault2.test 7b2741e5095367238380b0fcdb837f36c24484c7a5f353659b387df63 F test/fts3fault3.test 4a39a1618546776255dc1de306213b600aef87eca589ca8428a70c00fd11961b F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3fuzz001.test c78afcd8ad712ea0b8d2ed50851a8aab3bc9dc52c64a536291e07112f519357c +F test/fts3integrity.test 0c6fe7353d7b24d78862f4272ee9df4da2f32b3ff30fa3396945cda8119580a8 F test/fts3join.test 1a4d786539b2b79a41c28ef2ac22cacd92a8ee830249b68a7dee4a020848e3bb F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 F test/fts3matchinfo.test aa66cc50615578b30f6df9984819ae5b702511cf8a94251ec7c594096a703a4a @@ -2153,8 +2154,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b995aae510888a9746b46545d176a0885d4738e1f1bc0b7ad7937ed023efd7d6 -R e9d176533dea184874a9bb1447a17bcc -U drh -Z 1206a14cd99f29a5172802ae7d99e8c8 +P 23b92d915c12ee768857e2c3c961832f390cad9b53b8bcfc2b97664baab25bb7 +R c0f4a472eb421527038f0fdc417732a3 +U dan +Z f9a0b0cd41d3223ca7993a52ea0189d1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0d75d9cb1e..40eff92277 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23b92d915c12ee768857e2c3c961832f390cad9b53b8bcfc2b97664baab25bb7 \ No newline at end of file +88beb48472da4667c0727c8ebabe046ea526450ff837fe789d041ed3f1ff105e \ No newline at end of file diff --git a/src/vtab.c b/src/vtab.c index a9cfcb9d73..f839216787 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -315,7 +315,6 @@ void sqlite3VtabUnlockList(sqlite3 *db){ if( p ){ db->pDisconnect = 0; - sqlite3ExpirePreparedStatements(db, 0); do { VTable *pNext = p->pNext; sqlite3VtabUnlock(p); diff --git a/test/fts3integrity.test b/test/fts3integrity.test new file mode 100644 index 0000000000..bcbc49dc33 --- /dev/null +++ b/test/fts3integrity.test @@ -0,0 +1,42 @@ +# 2023 December 16 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file runs all tests. +# +# $Id: fts3.test,v 1.2 2008/07/23 18:17:32 drh Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix fts3integrity + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts3(x); + INSERT INTO t1 VALUES('first row'); + INSERT INTO t1 VALUES('second row'); + + CREATE TABLE t2(x PRIMARY KEY); + INSERT INTO t2 VALUES('first row'); + INSERT INTO t2 VALUES('second row'); +} + +sqlite3 db2 test.db + +do_execsql_test -db db2 1.1 { + CREATE TABLE t3(x, y); +} + +do_execsql_test 1.2 { + PRAGMA integrity_check; +} {ok} + +finish_test From 40e614e393268a6684d07b1a57ae9bb935d65b5a Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 15 Dec 2023 20:13:09 +0000 Subject: [PATCH 41/80] In the count-of-view optimization, deferring freeing obsolete parts of the parse tree, on the off-chance that some other part of the code might be holding a pointer to those parts. FossilOrigin-Name: da442578856c87137eb1677d9b13b7c1cf15828cc41d4756572b278060f69bae --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6a0e11442e..0051de08a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sexpiring\sprepared\sstatements\sin\sthe\smiddle\sof\san\sintegrity-check. -D 2023-12-15T19:26:16.801 +C In\sthe\scount-of-view\soptimization,\sdeferring\sfreeing\sobsolete\sparts\sof\sthe\nparse\stree,\son\sthe\soff-chance\sthat\ssome\sother\spart\sof\sthe\scode\smight\sbe\nholding\sa\spointer\sto\sthose\sparts. +D 2023-12-15T20:13:09.193 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -736,7 +736,7 @@ F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c da0a596a645cd7a8a9fd030e5126600b5639cc63f8ead18c1b67ff5a8a9b6a7f F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c f69e8e37c8bbe3bb02c890ad38a43c36757b87c7863d071d8ecd720732ed46cb +F src/select.c f1a81ff4f8e9e76c224e2ab3a4baa799add0db22158c7fcede65d8cc4a6fa2da F src/shell.c.in 0df801a0445af3fc55c7330bcd8396ca5433154bb0f728edc8be2d6f27764361 F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2154,8 +2154,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 23b92d915c12ee768857e2c3c961832f390cad9b53b8bcfc2b97664baab25bb7 -R c0f4a472eb421527038f0fdc417732a3 -U dan -Z f9a0b0cd41d3223ca7993a52ea0189d1 +P 88beb48472da4667c0727c8ebabe046ea526450ff837fe789d041ed3f1ff105e +R 9b65ab74c9a005ec2c0f339522792a80 +U drh +Z 954ebb3c0337beba214faae55272d5ff # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 40eff92277..1c58daef69 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88beb48472da4667c0727c8ebabe046ea526450ff837fe789d041ed3f1ff105e \ No newline at end of file +da442578856c87137eb1677d9b13b7c1cf15828cc41d4756572b278060f69bae \ No newline at end of file diff --git a/src/select.c b/src/select.c index d5b94e2da2..1215727791 100644 --- a/src/select.c +++ b/src/select.c @@ -7135,7 +7135,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ pSub->selFlags |= SF_Aggregate; pSub->selFlags &= ~SF_Compound; pSub->nSelectRow = 0; - sqlite3ExprListDelete(db, pSub->pEList); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pSub->pEList); pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount; pSub->pEList = sqlite3ExprListAppend(pParse, 0, pTerm); pTerm = sqlite3PExpr(pParse, TK_SELECT, 0, 0); From 5f3dd8668c21c1689238e3f340cc6b067805680c Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 16 Dec 2023 10:50:06 +0000 Subject: [PATCH 42/80] New test case based on Chromium bug report 1511689. FossilOrigin-Name: 2c7ef4b4d215f99f8d6787adb64e2037ae96e5dd6cb49c8b81634249f5e1b328 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/aggnested.test | 12 +++++++++++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0051de08a5..b600addb1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scount-of-view\soptimization,\sdeferring\sfreeing\sobsolete\sparts\sof\sthe\nparse\stree,\son\sthe\soff-chance\sthat\ssome\sother\spart\sof\sthe\scode\smight\sbe\nholding\sa\spointer\sto\sthose\sparts. -D 2023-12-15T20:13:09.193 +C New\stest\scase\sbased\son\sChromium\sbug\sreport\s1511689. +D 2023-12-16T10:50:06.926 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -831,7 +831,7 @@ F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867d F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggfault.test 777f269d0da5b0c2524c7ff6d99ae9a93db4f1b1839a914dd2a12e3035c29829 -F test/aggnested.test ce85a6af7d59c3109e35c5f03b2cd11da1a9b1417371e2f942102d0f0d77fd62 +F test/aggnested.test 610b0ce2c3e8f3daee25f9752800ee8d785db10da4aa1fbeea0ea1aabaf1d704 F test/aggorderby.test cc3abf5de64d46ff66395ca8c2346b66c2576d5aedb7bffc5b0742508856e3bf F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 2ecb8bbd52416642e41c9081182a8df05d42c75637afd4488aace78cc4b69e13 @@ -2154,8 +2154,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 88beb48472da4667c0727c8ebabe046ea526450ff837fe789d041ed3f1ff105e -R 9b65ab74c9a005ec2c0f339522792a80 +P da442578856c87137eb1677d9b13b7c1cf15828cc41d4756572b278060f69bae +R 7b23e47043e006e68f536d37e207685d U drh -Z 954ebb3c0337beba214faae55272d5ff +Z b495e05ec6841a5d9dda5bfbcac17be5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1c58daef69..c793e58bd8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da442578856c87137eb1677d9b13b7c1cf15828cc41d4756572b278060f69bae \ No newline at end of file +2c7ef4b4d215f99f8d6787adb64e2037ae96e5dd6cb49c8b81634249f5e1b328 \ No newline at end of file diff --git a/test/aggnested.test b/test/aggnested.test index ad6e208ddf..f3539076bd 100644 --- a/test/aggnested.test +++ b/test/aggnested.test @@ -472,6 +472,16 @@ do_execsql_test 9.5 { FROM t1; } {6} - +# 2023-12-16 +# New test case for check-in [4470f657d2069972] from 2023-11-02 +# https://bugs.chromium.org/p/chromium/issues/detail?id=1511689 +# +do_execsql_test 10.1 { + DROP TABLE IF EXISTS t0; + DROP TABLE IF EXISTS t1; + CREATE TABLE t0(c1, c2); INSERT INTO t0 VALUES(1,2); + CREATE TABLE t1(c3, c4); INSERT INTO t1 VALUES(3,4); + SELECT * FROM t0 WHERE EXISTS (SELECT 1 FROM t1 GROUP BY c3 HAVING ( SELECT count(*) FROM (SELECT 1 UNION ALL SELECT sum(DISTINCT c1) ) ) ) BETWEEN 1 AND 1; +} {1 2} finish_test From 13264418baadf2743c5bddbe5a1de272a7af42e0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 16 Dec 2023 15:48:42 +0000 Subject: [PATCH 43/80] Enable SQLITE_STRICT_SUBTYPE for default builds of the shell, fuzzcheck, and testfixture. FossilOrigin-Name: 5a0c517ed7e46c0f8a3db752cf5b9f8010c60f35084606abe9e7c1c4f993b4a7 --- Makefile.in | 5 ++++- Makefile.msc | 3 +++ autoconf/Makefile.msc | 1 + manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/test_func.c | 3 ++- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Makefile.in b/Makefile.in index b5c98fb8b8..5194253379 100644 --- a/Makefile.in +++ b/Makefile.in @@ -600,6 +600,7 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC +SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 FUZZERSHELL_OPT = FUZZCHECK_OPT += -I$(TOP)/test FUZZCHECK_OPT += -I$(TOP)/ext/recover @@ -630,7 +631,8 @@ FUZZCHECK_OPT += \ -DSQLITE_MAX_MMAP_SIZE=0 \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_PRINTF_PRECISION_LIMIT=1000 \ - -DSQLITE_PRIVATE="" + -DSQLITE_PRIVATE="" \ + -DSQLITE_STRICT_SUBTYPE=1 FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c @@ -1291,6 +1293,7 @@ TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_BYTECODE_VTAB TESTFIXTURE_FLAGS += -DSQLITE_CKSUMVFS_STATIC +TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la TESTFIXTURE_SRC1 = sqlite3.c diff --git a/Makefile.msc b/Makefile.msc index 3a4d46b01f..0e4a3f4845 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1692,6 +1692,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF # <> @@ -1728,6 +1729,7 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MMAP_SIZE=0 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE="" +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STRICT_SUBTYPE=1 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 @@ -2433,6 +2435,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CKSUMVFS_STATIC=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 280bb95deb..45a07a9f31 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -990,6 +990,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF diff --git a/manifest b/manifest index b600addb1d..8a8ef19428 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C New\stest\scase\sbased\son\sChromium\sbug\sreport\s1511689. -D 2023-12-16T10:50:06.926 +C Enable\sSQLITE_STRICT_SUBTYPE\sfor\sdefault\sbuilds\sof\sthe\sshell,\sfuzzcheck,\nand\stestfixture. +D 2023-12-16T15:48:42.762 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 890caf094636c308bc981032baf8f9208bf307755f9197ae4218a9fbcec2e449 +F Makefile.in c28e53bca825336006582531a2ae5b71de36839cab609d97de9a34991bb73a21 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 59bb36dba001f0b38212be0794fb838f25371008b180929bcf08aa799694c168 +F Makefile.msc a3b5113ad885bcdd67ff406b222dcf1ff78d2d449c1e0ba392c7477ff2db612c F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3 F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -15,7 +15,7 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am adedc1324b6a87fdd1265ddd336d2fb7d4f36a0e77b86ea553ae7cc4ea239347 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.msc 34f8c222846db1b354720c8a78a4fa7dec9176ea447c0fffdf442b7f74e8b1df +F autoconf/Makefile.msc ac338c36a338f6b49475da71930f45145a181d6b551578d5a7a64f113ef27b2c F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 42cfd21d0b19dc7d5d85fb5c405c5f3c6a4c923021c39128f6ba685355d8fd56 F autoconf/configure.ac ec7fa914c5e74ff212fe879f9bb6918e1234497e05facfb641f30c4d5893b277 @@ -765,7 +765,7 @@ F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf F src/test_demovfs.c 38a459d1c78fd9afa770445b224c485e079018d6ac07332ff9bd07b54d2b8ce9 F src/test_devsym.c 649434ed34d0b03fbd5a6b42df80f0f9a7e53f94dd1710aad5dd8831e91c4e86 F src/test_fs.c 56cc17e4fdc57efa61695026e2ba96e910b17060d7ee01d775ec048791522e2f -F src/test_func.c 24df3a346c012b1fc9e1001d346db6054deb426db0a7437e92490630e71c9b0a +F src/test_func.c 4d2dc7e3e0946e55091784ddaf0302294f2ee300614f6f3e19a4b38df77d5167 F src/test_hexio.c 9478e56a0f08e07841a014a93b20e4ba2709ab56d039d1ca8020e26846aa19bd F src/test_init.c f2cc4774b7c9140f76e45ecbb2ae219f68e3acbbe248c0179db666a70eae9f08 F src/test_intarray.c 26ffba666beb658d73cd925d9b4fb56913a3ca9aaeac122b3691436abb192b92 @@ -2154,8 +2154,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P da442578856c87137eb1677d9b13b7c1cf15828cc41d4756572b278060f69bae -R 7b23e47043e006e68f536d37e207685d +P 2c7ef4b4d215f99f8d6787adb64e2037ae96e5dd6cb49c8b81634249f5e1b328 +R 6b3ef196e12f3df9aad084402f2a2931 U drh -Z b495e05ec6841a5d9dda5bfbcac17be5 +Z 9f2e37a9158b0e1929ff27763f5e1a16 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c793e58bd8..4df31e16b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c7ef4b4d215f99f8d6787adb64e2037ae96e5dd6cb49c8b81634249f5e1b328 \ No newline at end of file +5a0c517ed7e46c0f8a3db752cf5b9f8010c60f35084606abe9e7c1c4f993b4a7 \ No newline at end of file diff --git a/src/test_func.c b/src/test_func.c index fa52438261..80df488282 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -694,7 +694,8 @@ static int registerTestFunctions( { "test_extract", 2, SQLITE_UTF8, test_extract}, { "test_zeroblob", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, test_zeroblob}, { "test_getsubtype", 1, SQLITE_UTF8, test_getsubtype}, - { "test_setsubtype", 2, SQLITE_UTF8, test_setsubtype}, + { "test_setsubtype", 2, SQLITE_UTF8|SQLITE_RESULT_SUBTYPE, + test_setsubtype}, { "test_frombind", -1, SQLITE_UTF8, test_frombind}, }; int i; From 0d201598a50a7dcf96de6d60ddb73488f6c13abb Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 17 Dec 2023 20:41:48 +0000 Subject: [PATCH 44/80] Enhancements to the "randomjson.c" extension. Automatically load that extension into fuzzcheck. FossilOrigin-Name: 70620405ab01d6a5d38bafa9ae175fd6e4eabaf2efb7854734278dafd7b05c99 --- Makefile.in | 4 +++- Makefile.msc | 2 ++ ext/misc/randomjson.c | 24 ++++++++++++++++-------- main.mk | 3 +++ manifest | 20 ++++++++++---------- manifest.uuid | 2 +- test/fuzzcheck.c | 6 +++++- 7 files changed, 40 insertions(+), 21 deletions(-) diff --git a/Makefile.in b/Makefile.in index 5194253379..a461685bf2 100644 --- a/Makefile.in +++ b/Makefile.in @@ -632,7 +632,8 @@ FUZZCHECK_OPT += \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_PRINTF_PRECISION_LIMIT=1000 \ -DSQLITE_PRIVATE="" \ - -DSQLITE_STRICT_SUBTYPE=1 + -DSQLITE_STRICT_SUBTYPE=1 \ + -DSQLITE_STATIC_RANDOMJSON FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c @@ -640,6 +641,7 @@ FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c FUZZCHECK_SRC += $(TOP)/test/vt02.c +FUZZCHECK_SRC += $(TOP)/ext/misc/randomjson.c DBFUZZ_OPT = ST_OPT = -DSQLITE_OS_KV_OPTIONAL diff --git a/Makefile.msc b/Makefile.msc index 0e4a3f4845..81ee6bb561 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1730,6 +1730,7 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE="" FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STRICT_SUBTYPE=1 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STATIC_RANDOMJSON FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 @@ -1746,6 +1747,7 @@ FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\randomjson.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION diff --git a/ext/misc/randomjson.c b/ext/misc/randomjson.c index 3a6f545fe6..a44905ca88 100644 --- a/ext/misc/randomjson.c +++ b/ext/misc/randomjson.c @@ -27,8 +27,12 @@ ** .load ./randomjson ** SELECT random_json(1); */ -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 +#ifdef SQLITE_STATIC_RANDOMJSON +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif #include #include #include @@ -52,7 +56,7 @@ static unsigned int prngInt(Prng *p){ } static const char *azJsonAtoms[] = { - /* JSON /* JSON-5 */ + /* JSON JSON-5 */ "0", "0", "1", "1", "-1", "-1", @@ -95,13 +99,13 @@ static const char *azJsonTemplate[] = { /* JSON JSON-5 */ "{\"a\":%,\"b\":%,\"c\":%}", "{a:%,b:%,c:%}", "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"e\":%}", "{a:%,b:%,c:%,d:%,e:%}", - "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}", "{a:%,b:%,c:%,d:%,\"\":%}", + "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}", "{a:%,b:%,c:%,d:%,'':%}", "{\"d\":%}", "{d:%}", "{\"eeee\":%, \"ffff\":%}", "{eeee:% /*and*/, ffff:%}", "{\"$g\":%,\"_h_\":%}", "{$g:%,_h_:%,}", "{\"x\":%,\n \"y\":%}", "{\"x\":%,\n \"y\":%}", - "{\"a b c d\":%,\"e\":%,\"f\":%,\"x\":%,\"y\":%}", - "{\"a b c d\":%,e:%,f:%,x:%,y:%}", + "{\"a b c d\":%,\"e\":%,\"f\":%,\"\\u0078\":%,\"y\":%}", + "{\"a b c d\":%,\"\\x65\":%,\"\\u0066\":%,x:%,y:%}", "{\"Z\":%}", "{Z:%,}", "[%]", "[%,]", "[%,%]", "[%,%]", @@ -151,7 +155,7 @@ static void jsonExpand( n = strlen(z); if( j+n Date: Mon, 18 Dec 2023 12:18:47 +0000 Subject: [PATCH 45/80] Enhancements to ext/misc/randomjson.c. FossilOrigin-Name: a4e6d1f86f3a502e4170f5a90031e269e48363e95114a66b84d373e3ce0b2704 --- ext/misc/randomjson.c | 47 ++++++++++++++++++++++++++++++++++--------- manifest | 12 +++++------ manifest.uuid | 2 +- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/ext/misc/randomjson.c b/ext/misc/randomjson.c index a44905ca88..d64eb69327 100644 --- a/ext/misc/randomjson.c +++ b/ext/misc/randomjson.c @@ -26,6 +26,7 @@ ** ** .load ./randomjson ** SELECT random_json(1); +** SELECT random_json5(1); */ #ifdef SQLITE_STATIC_RANDOMJSON # include "sqlite3.h" @@ -55,17 +56,18 @@ static unsigned int prngInt(Prng *p){ return p->x ^ p->y; } -static const char *azJsonAtoms[] = { +static char *azJsonAtoms[] = { /* JSON JSON-5 */ "0", "0", "1", "1", "-1", "-1", "2", "+2", - "3", "3", - "2.5", "2.5", + "3DDDD", "3DDDD", + "2.5DD", "2.5DD", "0.75", ".75", "-4.0e2", "-4.e2", "5.0e-3", "+5e-3", + "6.DDe+0DD", "6.DDe+0DD", "0", "0x0", "512", "0x200", "256", "+0x100", @@ -77,12 +79,14 @@ static const char *azJsonAtoms[] = { "-9.0e999", "-Infinity", "9.0e999", "+Infinity", "null", "NaN", - "-0.0005123", "-0.0005123", + "-0.0005DD", "-0.0005DD", "4.35e-3", "+4.35e-3", "\"gem\\\"hay\"", "\"gem\\\"hay\"", "\"icy'joy\"", "'icy\\'joy\'", "\"keylog\"", "\"key\\\nlog\"", "\"mix\\\\\\tnet\"", "\"mix\\\\\\tnet\"", + "\"oat\\r\\n\"", "\"oat\\r\\n\"", + "\"\\fpan\\b\"", "\"\\fpan\\b\"", "{}", "{}", "[]", "[]", "[]", "[/*empty*/]", @@ -93,19 +97,21 @@ static const char *azJsonAtoms[] = { "\"day\"", "\"day\"", "\"end\"", "'end'", "\"fly\"", "\"fly\"", + "\"\\u00XX\\u00XX\"", "\"\\xXX\\xXX\"", + "\"y\\uXXXXz\"", "\"y\\uXXXXz\"", "\"\"", "\"\"", }; -static const char *azJsonTemplate[] = { +static char *azJsonTemplate[] = { /* JSON JSON-5 */ - "{\"a\":%,\"b\":%,\"c\":%}", "{a:%,b:%,c:%}", + "{\"a\":%,\"b\":%,\"cDD\":%}", "{a:%,b:%,cDD:%}", "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"e\":%}", "{a:%,b:%,c:%,d:%,e:%}", "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}", "{a:%,b:%,c:%,d:%,'':%}", "{\"d\":%}", "{d:%}", "{\"eeee\":%, \"ffff\":%}", "{eeee:% /*and*/, ffff:%}", "{\"$g\":%,\"_h_\":%}", "{$g:%,_h_:%,}", "{\"x\":%,\n \"y\":%}", "{\"x\":%,\n \"y\":%}", - "{\"a b c d\":%,\"e\":%,\"f\":%,\"\\u0078\":%,\"y\":%}", - "{\"a b c d\":%,\"\\x65\":%,\"\\u0066\":%,x:%,y:%}", + "{\"a b c d\":%,\"\\u00XX\":%,\"\\uXXXX\":%,\"x\":%,\"y\":%}", + "{\"a b c d\":%,\"\\xXX\":%,\"\\uXXXX\":%,x:%,y:%}", "{\"Z\":%}", "{Z:%,}", "[%]", "[%,]", "[%,%]", "[%,%]", @@ -126,8 +132,10 @@ static void jsonExpand( unsigned int r /* Growth probability 0..1000. 0 means no growth */ ){ unsigned int i, j, k; - const char *z; + char *z; + char *zX; size_t n; + char zBuf[200]; j = 0; if( zSrc==0 ){ @@ -153,6 +161,27 @@ static void jsonExpand( z = azJsonTemplate[k]; } n = strlen(z); + if( (zX = strstr(z,"XX"))!=0 ){ + unsigned int r = prngInt(p); + memcpy(zBuf, z, n+1); + z = zBuf; + zX = strstr(z,"XX"); + while( zX!=0 ){ + zX[0] = "0123456789abcdef"[r%16]; r /= 16; + zX[1] = "0123456789abcdef"[r%16]; r /= 16; + zX = strstr(zX, "XX"); + } + }else if( (zX = strstr(z,"DD"))!=0 ){ + unsigned int r = prngInt(p); + memcpy(zBuf, z, n+1); + z = zBuf; + zX = strstr(z,"DD"); + while( zX!=0 ){ + zX[0] = "0123456789"[r%10]; r /= 10; + zX[1] = "0123456789"[r%10]; r /= 10; + zX = strstr(zX, "DD"); + } + } if( j+n Date: Mon, 18 Dec 2023 13:51:54 +0000 Subject: [PATCH 46/80] Bug fix in the randomjson.c extension. FossilOrigin-Name: 1f3a33df530dbe330ea8b14a69369b807b413b25a167d1a3938f8f0faf97cc91 --- ext/misc/randomjson.c | 8 +++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ext/misc/randomjson.c b/ext/misc/randomjson.c index d64eb69327..20ee12b9fe 100644 --- a/ext/misc/randomjson.c +++ b/ext/misc/randomjson.c @@ -138,11 +138,7 @@ static void jsonExpand( char zBuf[200]; j = 0; - if( zSrc==0 ){ - k = prngInt(p)%(count(azJsonTemplate)/2); - k = k*2 + eType; - zSrc = azJsonTemplate[k]; - } + if( zSrc==0 ) zSrc = "%"; if( strlen(zSrc)>=STRSZ/10 ) r = 0; for(i=0; zSrc[i]; i++){ if( zSrc[i]!='%' ){ @@ -182,6 +178,8 @@ static void jsonExpand( zX = strstr(zX, "DD"); } } + assert( strstr(z, "XX")==0 ); + assert( strstr(z, "DD")==0 ); if( j+n Date: Mon, 18 Dec 2023 14:16:58 +0000 Subject: [PATCH 47/80] Ensure that all object labels for individual objects generated by randomjson.c are unique. FossilOrigin-Name: 29c46aca231b3f1e997ef306a5a651408185bf3ad09ab9fc1fe21ed18caa4d02 --- ext/misc/randomjson.c | 9 ++++++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ext/misc/randomjson.c b/ext/misc/randomjson.c index 20ee12b9fe..5036fa6047 100644 --- a/ext/misc/randomjson.c +++ b/ext/misc/randomjson.c @@ -108,10 +108,9 @@ static char *azJsonTemplate[] = { "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}", "{a:%,b:%,c:%,d:%,'':%}", "{\"d\":%}", "{d:%}", "{\"eeee\":%, \"ffff\":%}", "{eeee:% /*and*/, ffff:%}", - "{\"$g\":%,\"_h_\":%}", "{$g:%,_h_:%,}", + "{\"$g\":%,\"_h_\":%,\"a b c d\":%}", "{$g:%,_h_:%,\"a b c d\":%}", "{\"x\":%,\n \"y\":%}", "{\"x\":%,\n \"y\":%}", - "{\"a b c d\":%,\"\\u00XX\":%,\"\\uXXXX\":%,\"x\":%,\"y\":%}", - "{\"a b c d\":%,\"\\xXX\":%,\"\\uXXXX\":%,x:%,y:%}", + "{\"\\u00XX\":%,\"\\uXXXX\":%}", "{\"\\xXX\":%,\"\\uXXXX\":%}", "{\"Z\":%}", "{Z:%,}", "[%]", "[%,]", "[%,%]", "[%,%]", @@ -159,6 +158,10 @@ static void jsonExpand( n = strlen(z); if( (zX = strstr(z,"XX"))!=0 ){ unsigned int r = prngInt(p); + if( (r&0xff)==((r>>8)&0xff) ) r += 0x100; + while( (r&0xff)==((r>>16)&0xff) || ((r>>8)&0xff)==((r>>16)&0xff) ){ + r += 0x10000; + } memcpy(zBuf, z, n+1); z = zBuf; zX = strstr(z,"XX"); diff --git a/manifest b/manifest index 7141e85632..c15311e551 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bug\sfix\sin\sthe\srandomjson.c\sextension. -D 2023-12-18T13:51:54.496 +C Ensure\sthat\sall\sobject\slabels\sfor\sindividual\sobjects\sgenerated\sby\nrandomjson.c\sare\sunique. +D 2023-12-18T14:16:58.155 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -394,7 +394,7 @@ F ext/misc/pcachetrace.c f4227ce03fb16aa8d6f321b72dd051097419d7a028a9853af048bee F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c -F ext/misc/randomjson.c 8642d0ce204de743f148c11b320f4493046bf2a57f79efb19be3388101a7d092 +F ext/misc/randomjson.c 182cabbeb294f56f68ba2fd0879790e6d4889bb50b6a532604cc6f360d6dd516 F ext/misc/regexp.c 4bdd0045912f81c84908bd535ec5ad3b1c8540b4287c70ab84070963624047db F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c @@ -2154,8 +2154,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4e6d1f86f3a502e4170f5a90031e269e48363e95114a66b84d373e3ce0b2704 -R 9787f93b49de93cd2fa8fe010120ae0d +P 1f3a33df530dbe330ea8b14a69369b807b413b25a167d1a3938f8f0faf97cc91 +R 52287023e6071fa0fbc19b0133af9058 U drh -Z 72d64dbdca15edf4ed5cc684c21af296 +Z 09d32bf163b956d3c1ed37f12088104f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c6d3735f6a..90e206a8b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f3a33df530dbe330ea8b14a69369b807b413b25a167d1a3938f8f0faf97cc91 \ No newline at end of file +29c46aca231b3f1e997ef306a5a651408185bf3ad09ab9fc1fe21ed18caa4d02 \ No newline at end of file From cf6fe5abe31020a895b34502bad3deb6708fdac7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 18 Dec 2023 14:24:13 +0000 Subject: [PATCH 48/80] Add randomjson.c to testfixture. Use it for a new set of invariant tests against JSON functions. FossilOrigin-Name: f1c040606bfe784804134d8f3ca130908fad5212b47e3c32792baab977470943 --- Makefile.in | 2 ++ Makefile.msc | 2 ++ main.mk | 3 +++ manifest | 21 +++++++++++---------- manifest.uuid | 2 +- src/test1.c | 2 ++ test/json105.test | 2 +- test/json106.test | 44 ++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 test/json106.test diff --git a/Makefile.in b/Makefile.in index a461685bf2..cb894666d9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -447,6 +447,7 @@ TESTSRC += \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/prefixes.c \ $(TOP)/ext/misc/qpvtab.c \ + $(TOP)/ext/misc/randomjson.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ @@ -1295,6 +1296,7 @@ TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_BYTECODE_VTAB TESTFIXTURE_FLAGS += -DSQLITE_CKSUMVFS_STATIC +TESTFIXTURE_FLAGS += -DSQLITE_STATIC_RANDOMJSON TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la diff --git a/Makefile.msc b/Makefile.msc index 81ee6bb561..15c44b62f9 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1584,6 +1584,7 @@ TESTEXT = \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\prefixes.c \ $(TOP)\ext\misc\qpvtab.c \ + $(TOP)\ext\misc\randomjson.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\remember.c \ $(TOP)\ext\misc\series.c \ @@ -2437,6 +2438,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CKSUMVFS_STATIC=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STATIC_RANDOMJSON TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) diff --git a/main.mk b/main.mk index f746b1bb67..081e0cd3b5 100644 --- a/main.mk +++ b/main.mk @@ -360,6 +360,7 @@ TESTSRC += \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/prefixes.c \ $(TOP)/ext/misc/qpvtab.c \ + $(TOP)/ext/misc/randomjson.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ @@ -901,6 +902,8 @@ TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_BYTECODE_VTAB TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit TESTFIXTURE_FLAGS += -DSQLITE_CKSUMVFS_STATIC +TESTFIXTURE_FLAGS += -DSQLITE_STATIC_RANDOMJSON +TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1 testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ diff --git a/manifest b/manifest index c15311e551..b37b609b85 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Ensure\sthat\sall\sobject\slabels\sfor\sindividual\sobjects\sgenerated\sby\nrandomjson.c\sare\sunique. -D 2023-12-18T14:16:58.155 +C Add\srandomjson.c\sto\stestfixture.\s\sUse\sit\sfor\sa\snew\sset\sof\sinvariant\stests\nagainst\sJSON\sfunctions. +D 2023-12-18T14:24:13.030 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 8e458bac73ffdbd46bc486d243a9aebb589fca36c189bac7ae6c7a53c4013b22 +F Makefile.in 24be65ae641c5727bbc247d60286a15ecc24fb80f14f8fb2d32533bf0ec96e79 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 2b563f70c18211698d9fc7720007cde52f37549e0802e172ae11d022ced3fcf8 +F Makefile.msc ddca98e096947fc26e3bb1a20678999dd0a590456278ad7dc55b882fca108bd6 F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3 F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -656,7 +656,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk c6690ad32427ca98ba66e13a4747f3bcf644b2973e2a3a9d8921db7e163055a3 +F main.mk ef8d6b0e76b27d2b32d7b71ea30a2b2626b668f998a4f32f6171c9623a310a53 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -746,7 +746,7 @@ F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c ecbc3c99c0d0c3ed122a913f143026c26d38d57f33e06bb71185dd5c1efe37cd -F src/test1.c f9620e8f0d0fa4edb239201a732c4dd1562f0cdd9741955c89332d49e14a5edd +F src/test1.c ac6542cddd1f405e332d869946b977b2ce8b4dc28b9f7cc61df38abe1fe49bc3 F src/test2.c 54520d0565ef2b9bf0f8f1dcac43dc4d06baf4ffe13d10905f8d8c3ad3e4b9ab F src/test3.c e5178558c41ff53236ae0271e9acb3d6885a94981d2eb939536ee6474598840e F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 @@ -1338,7 +1338,8 @@ F test/json101.test 70587d7d35ef9e2126364ba70f0c951f70827cfbd28649d779ff3df7e8f8 F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 -F test/json105.test e64a8d73677fbae67886642cd5076e2ef3efe89f8483b87595cf9c030216c9bd +F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988 +F test/json106.test 71008c4341539445e85a6124dc77a2938c9a9b175af30826a50975e8fe79a8dc F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330 F test/json502.test 84634d3dbb521d2814e43624025b760c6198456c8197bbec6c977c0236648f5b F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 @@ -2154,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1f3a33df530dbe330ea8b14a69369b807b413b25a167d1a3938f8f0faf97cc91 -R 52287023e6071fa0fbc19b0133af9058 +P 29c46aca231b3f1e997ef306a5a651408185bf3ad09ab9fc1fe21ed18caa4d02 +R 3b15382c1a769a7c8cb14a6589eb5342 U drh -Z 09d32bf163b956d3c1ed37f12088104f +Z 9a206cabf99e0d7c9b2e0213613cb0fd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 90e206a8b2..3f84658ba8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29c46aca231b3f1e997ef306a5a651408185bf3ad09ab9fc1fe21ed18caa4d02 \ No newline at end of file +f1c040606bfe784804134d8f3ca130908fad5212b47e3c32792baab977470943 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 55be0596b0..9c28259b41 100644 --- a/src/test1.c +++ b/src/test1.c @@ -8120,6 +8120,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd( extern int sqlite3_prefixes_init(sqlite3*,char**,const sqlite3_api_routines*); #endif extern int sqlite3_qpvtab_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_randomjson_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_remember_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_series_init(sqlite3*,char**,const sqlite3_api_routines*); @@ -8152,6 +8153,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd( { "prefixes", sqlite3_prefixes_init }, #endif { "qpvtab", sqlite3_qpvtab_init }, + { "randomjson", sqlite3_randomjson_init }, { "regexp", sqlite3_regexp_init }, { "remember", sqlite3_remember_init }, { "series", sqlite3_series_init }, diff --git a/test/json105.test b/test/json105.test index 5592191c22..509db94e11 100644 --- a/test/json105.test +++ b/test/json105.test @@ -13,7 +13,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix json104 +set testprefix json105 # This is the example from pages 2 and 3 of RFC-7396 db eval { diff --git a/test/json106.test b/test/json106.test new file mode 100644 index 0000000000..e5a1ac5bc4 --- /dev/null +++ b/test/json106.test @@ -0,0 +1,44 @@ +# 2023-12-18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Invariant tests for JSON built around the randomjson extension +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json106 + +load_static_extension db randomjson +db eval { + CREATE TEMP TABLE t1(j0,j5); +} +unset -nocomplain ii +for {set ii 1} {$ii<=5000} {incr ii} { + do_execsql_test $ii.1 { + DELETE FROM t1; + INSERT INTO t1(j0,j5) VALUES(random_json($ii),random_json5($ii)); + SELECT json_valid(j0), json_valid(j5,2) FROM t1; + } {1 1} + do_execsql_test $ii.2 { + SELECT count(*) + FROM t1, json_tree(j0) AS rt + WHERE rt.type NOT IN ('object','array') + AND rt.atom IS NOT (j0 ->> rt.fullkey); + } 0 + do_execsql_test $ii.3 { + SELECT count(*) + FROM t1, json_tree(j5) AS rt + WHERE rt.type NOT IN ('object','array') + AND rt.atom IS NOT (j0 ->> rt.fullkey); + } 0 +} + + +finish_test From 095f2c5082e85d8122225227d3c8923880ef348d Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 18 Dec 2023 15:53:48 +0000 Subject: [PATCH 49/80] Ensure that the insert/delete size delta on JSONB objects in the JSON cache are always set to zero. FossilOrigin-Name: 4b4581668a908473dbf1322a3e98bc7cca122998c44518ea183af7f0d1ba9f95 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b37b609b85..3d39d056ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\srandomjson.c\sto\stestfixture.\s\sUse\sit\sfor\sa\snew\sset\sof\sinvariant\stests\nagainst\sJSON\sfunctions. -D 2023-12-18T14:24:13.030 +C Ensure\sthat\sthe\sinsert/delete\ssize\sdelta\son\sJSONB\sobjects\sin\sthe\sJSON\scache\nare\salways\sset\sto\szero. +D 2023-12-18T15:53:48.421 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 55aaffca9b1a136ea5937564f2159f300c2d7c3711cdb1c23356a7d27885827b +F src/json.c 953ac749c4d348ddbcbb2851a66188d6ff9928ddc1daa7a56f8555fc1cecb806 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 29c46aca231b3f1e997ef306a5a651408185bf3ad09ab9fc1fe21ed18caa4d02 -R 3b15382c1a769a7c8cb14a6589eb5342 +P f1c040606bfe784804134d8f3ca130908fad5212b47e3c32792baab977470943 +R 01dea41071f1e54cd13e89135ed2f4ac U drh -Z 9a206cabf99e0d7c9b2e0213613cb0fd +Z 65bb882629df28755f3c8ef3dc1544eb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f84658ba8..3f29a4333a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1c040606bfe784804134d8f3ca130908fad5212b47e3c32792baab977470943 \ No newline at end of file +4b4581668a908473dbf1322a3e98bc7cca122998c44518ea183af7f0d1ba9f95 \ No newline at end of file diff --git a/src/json.c b/src/json.c index a1441e8099..2756590eb7 100644 --- a/src/json.c +++ b/src/json.c @@ -374,6 +374,7 @@ static int jsonCacheInsert( assert( pParse->zJson!=0 ); assert( pParse->bJsonIsRCStr ); + assert( pParse->delta==0 ); p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); if( p==0 ){ sqlite3 *db = sqlite3_context_db_handle(ctx); @@ -446,6 +447,7 @@ static JsonParse *jsonCacheSearch( p->a[p->nUsed-1] = tmp; i = p->nUsed - 1; } + assert( p->a[i]->delta==0 ); return p->a[i]; }else{ return 0; @@ -3306,6 +3308,7 @@ static void jsonReturnParse( }else{ JsonString s; jsonStringInit(&s, ctx); + p->delta = 0; jsonXlateBlobToText(p, 0, &s); jsonReturnString(&s, p, ctx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); From fc76750f616a8ceadc9988de67e065b6e274547a Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 18 Dec 2023 18:31:27 +0000 Subject: [PATCH 50/80] Fix JSON to JSONB translation so that it deals correctly with Infinity and NaN. FossilOrigin-Name: 178cb84f36bdb45ba17511900d6d8ea8dfa14912fc5bf7094a20348174a36c95 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 22 +++++++++++++++++++--- test/json106.test | 20 ++++++++++++++++++++ 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 3d39d056ef..3b2af6b615 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\sinsert/delete\ssize\sdelta\son\sJSONB\sobjects\sin\sthe\sJSON\scache\nare\salways\sset\sto\szero. -D 2023-12-18T15:53:48.421 +C Fix\sJSON\sto\sJSONB\stranslation\sso\sthat\sit\sdeals\scorrectly\swith\sInfinity\nand\sNaN. +D 2023-12-18T18:31:27.729 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 953ac749c4d348ddbcbb2851a66188d6ff9928ddc1daa7a56f8555fc1cecb806 +F src/json.c f405fac347de6f1fb5fa1eb92616499dd333992f4db524e7bef4604899bf20ba F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -1339,7 +1339,7 @@ F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988 -F test/json106.test 71008c4341539445e85a6124dc77a2938c9a9b175af30826a50975e8fe79a8dc +F test/json106.test 461e02e1f0cd2d5be1954c8888d2e771c2adf46df4c075b6c5da1bde66eed010 F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330 F test/json502.test 84634d3dbb521d2814e43624025b760c6198456c8197bbec6c977c0236648f5b F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f1c040606bfe784804134d8f3ca130908fad5212b47e3c32792baab977470943 -R 01dea41071f1e54cd13e89135ed2f4ac +P 4b4581668a908473dbf1322a3e98bc7cca122998c44518ea183af7f0d1ba9f95 +R 9c86ed12e78ae744d874c37c356e4dab U drh -Z 65bb882629df28755f3c8ef3dc1544eb +Z ffac4918f7e206860fa3115d6182bb2e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f29a4333a..52924a1d55 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b4581668a908473dbf1322a3e98bc7cca122998c44518ea183af7f0d1ba9f95 \ No newline at end of file +178cb84f36bdb45ba17511900d6d8ea8dfa14912fc5bf7094a20348174a36c95 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 2756590eb7..b4b9d8f81e 100644 --- a/src/json.c +++ b/src/json.c @@ -3041,13 +3041,29 @@ static int jsonFunctionArgToBlob( } break; } - case SQLITE_FLOAT: + case SQLITE_FLOAT: { + double r = sqlite3_value_double(pArg); + if( sqlite3IsNaN(r) ){ + jsonBlobAppendNode(pParse, JSONB_NULL, 0, 0); + }else{ + int n = sqlite3_value_bytes(pArg); + const char *z = (const char*)sqlite3_value_text(pArg); + if( z==0 ) return 1; + if( z[0]=='I' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); + }else if( z[0]=='-' && z[1]=='I' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 6, "-9e999"); + }else{ + jsonBlobAppendNode(pParse, JSONB_FLOAT, n, z); + } + } + break; + } case SQLITE_INTEGER: { int n = sqlite3_value_bytes(pArg); const char *z = (const char*)sqlite3_value_text(pArg); - int e = eType==SQLITE_INTEGER ? JSONB_INT : JSONB_FLOAT; if( z==0 ) return 1; - jsonBlobAppendNode(pParse, e, n, z); + jsonBlobAppendNode(pParse, JSONB_INT, n, z); break; } } diff --git a/test/json106.test b/test/json106.test index e5a1ac5bc4..ba70d8af5f 100644 --- a/test/json106.test +++ b/test/json106.test @@ -18,6 +18,7 @@ set testprefix json106 load_static_extension db randomjson db eval { CREATE TEMP TABLE t1(j0,j5); + CREATE TEMP TABLE kv(n,key,val); } unset -nocomplain ii for {set ii 1} {$ii<=5000} {incr ii} { @@ -38,6 +39,25 @@ for {set ii 1} {$ii<=5000} {incr ii} { WHERE rt.type NOT IN ('object','array') AND rt.atom IS NOT (j0 ->> rt.fullkey); } 0 + do_execsql_test $ii.4 { + DELETE FROM kv; + INSERT INTO kv + SELECT rt.rowid, rt.fullkey, rt.atom + FROM t1, json_tree(j0) AS rt + WHERE rt.type NOT IN ('object','array'); + } + do_execsql_test $ii.5 { + SELECT count(*) + FROM t1, kv + WHERE key NOT LIKE '%]' + AND json_remove(j5,key)->>key IS NOT NULL + } 0 + do_execsql_test $ii.6 { + SELECT count(*) + FROM t1, kv + WHERE key NOT LIKE '%]' + AND json_insert(json_remove(j5,key),key,val)->>key IS NOT val + } 0 } From c4dd6b42ec50d7b176a591d21044d6cd4ebaa782 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 18 Dec 2023 18:50:47 +0000 Subject: [PATCH 51/80] Add NEVER() to an unfalsifiable branch. FossilOrigin-Name: 9a0c67db366d38a0b0741f6a1ae333cf27cfe6f6b7c6eed94bdec9686f9f9f8a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3b2af6b615..d814b23985 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sJSON\sto\sJSONB\stranslation\sso\sthat\sit\sdeals\scorrectly\swith\sInfinity\nand\sNaN. -D 2023-12-18T18:31:27.729 +C Add\sNEVER()\sto\san\sunfalsifiable\sbranch. +D 2023-12-18T18:50:47.524 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c f405fac347de6f1fb5fa1eb92616499dd333992f4db524e7bef4604899bf20ba +F src/json.c 7200bf814ea6766ebc2fb236c151e4f4b679b53e564cedf99ea23eb22d1c7db1 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4b4581668a908473dbf1322a3e98bc7cca122998c44518ea183af7f0d1ba9f95 -R 9c86ed12e78ae744d874c37c356e4dab +P 178cb84f36bdb45ba17511900d6d8ea8dfa14912fc5bf7094a20348174a36c95 +R b00d696eab2232f0273569f0a1e137d7 U drh -Z ffac4918f7e206860fa3115d6182bb2e +Z 5f8069f1653cd9e30ea4568179380c08 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 52924a1d55..372f35b7a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -178cb84f36bdb45ba17511900d6d8ea8dfa14912fc5bf7094a20348174a36c95 \ No newline at end of file +9a0c67db366d38a0b0741f6a1ae333cf27cfe6f6b7c6eed94bdec9686f9f9f8a \ No newline at end of file diff --git a/src/json.c b/src/json.c index b4b9d8f81e..8f7938a23e 100644 --- a/src/json.c +++ b/src/json.c @@ -3043,7 +3043,7 @@ static int jsonFunctionArgToBlob( } case SQLITE_FLOAT: { double r = sqlite3_value_double(pArg); - if( sqlite3IsNaN(r) ){ + if( NEVER(sqlite3IsNaN(r)) ){ jsonBlobAppendNode(pParse, JSONB_NULL, 0, 0); }else{ int n = sqlite3_value_bytes(pArg); From d9252d09c2e5a850636ff3103196d0af028ecfaa Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 18 Dec 2023 19:18:46 +0000 Subject: [PATCH 52/80] New JSON invariant test cases. FossilOrigin-Name: a6a1367b0bf364b1a2e20e153c5f4a578624b8846f9ec0b7c9c3cba0ea2ec346 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/json106.test | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d814b23985..33bc9e1519 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sNEVER()\sto\san\sunfalsifiable\sbranch. -D 2023-12-18T18:50:47.524 +C New\sJSON\sinvariant\stest\scases. +D 2023-12-18T19:18:46.614 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1339,7 +1339,7 @@ F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988 -F test/json106.test 461e02e1f0cd2d5be1954c8888d2e771c2adf46df4c075b6c5da1bde66eed010 +F test/json106.test dfd32e3bc829cd7068c98830f423d6e24cb55059ae3e5350e9a8a6d9bc4b6412 F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330 F test/json502.test 84634d3dbb521d2814e43624025b760c6198456c8197bbec6c977c0236648f5b F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 178cb84f36bdb45ba17511900d6d8ea8dfa14912fc5bf7094a20348174a36c95 -R b00d696eab2232f0273569f0a1e137d7 +P 9a0c67db366d38a0b0741f6a1ae333cf27cfe6f6b7c6eed94bdec9686f9f9f8a +R 561c0542979adb78774743e8741313a7 U drh -Z 5f8069f1653cd9e30ea4568179380c08 +Z e8c49215c95f79377b07d9e516c1b219 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 372f35b7a6..ecf906dbda 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a0c67db366d38a0b0741f6a1ae333cf27cfe6f6b7c6eed94bdec9686f9f9f8a \ No newline at end of file +a6a1367b0bf364b1a2e20e153c5f4a578624b8846f9ec0b7c9c3cba0ea2ec346 \ No newline at end of file diff --git a/test/json106.test b/test/json106.test index ba70d8af5f..f52d2b6386 100644 --- a/test/json106.test +++ b/test/json106.test @@ -17,7 +17,7 @@ set testprefix json106 load_static_extension db randomjson db eval { - CREATE TEMP TABLE t1(j0,j5); + CREATE TEMP TABLE t1(j0,j5,p); CREATE TEMP TABLE kv(n,key,val); } unset -nocomplain ii @@ -58,6 +58,12 @@ for {set ii 1} {$ii<=5000} {incr ii} { WHERE key NOT LIKE '%]' AND json_insert(json_remove(j5,key),key,val)->>key IS NOT val } 0 + do_execsql_test $ii.7 { + UPDATE t1 SET p=json_patch(j0,j5); + SELECT count(*) + FROM t1, kv + WHERE p->>key IS NOT val + } 0 } From 459f7b109014849034b930bc950cfff1f93b6dcf Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 00:07:38 +0000 Subject: [PATCH 53/80] Remove a stray comment in the JSON code. FossilOrigin-Name: 6618bdf0679405b43911ea8cd94050b12a5dc469f3dfe4759ee3ff850a55229e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 5 ----- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 33bc9e1519..a10a16749a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sJSON\sinvariant\stest\scases. -D 2023-12-18T19:18:46.614 +C Remove\sa\sstray\scomment\sin\sthe\sJSON\scode. +D 2023-12-19T00:07:38.941 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 7200bf814ea6766ebc2fb236c151e4f4b679b53e564cedf99ea23eb22d1c7db1 +F src/json.c 661b6da631185152c635161d52c274255c8de3f5769570bbbde0311cd9ad7460 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9a0c67db366d38a0b0741f6a1ae333cf27cfe6f6b7c6eed94bdec9686f9f9f8a -R 561c0542979adb78774743e8741313a7 +P a6a1367b0bf364b1a2e20e153c5f4a578624b8846f9ec0b7c9c3cba0ea2ec346 +R 8cbf9e55e2fcc826ff07211e47ebf076 U drh -Z e8c49215c95f79377b07d9e516c1b219 +Z 1332cee5da58f474b98f2121b7eff333 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ecf906dbda..0ba65215cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6a1367b0bf364b1a2e20e153c5f4a578624b8846f9ec0b7c9c3cba0ea2ec346 \ No newline at end of file +6618bdf0679405b43911ea8cd94050b12a5dc469f3dfe4759ee3ff850a55229e \ No newline at end of file diff --git a/src/json.c b/src/json.c index 8f7938a23e..d0e7b3fc6b 100644 --- a/src/json.c +++ b/src/json.c @@ -3165,11 +3165,6 @@ jsonInsertIntoBlob_patherror: return; } -/* -** Make a copy of a JsonParse object. The copy will be editable. -*/ - - /* ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, ** from the SQL function argument pArg. Return a pointer to the new From a3569fb2ca818cfdb63296413949a6169c62b086 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 11:57:29 +0000 Subject: [PATCH 54/80] Extra ALWAYS() macros to verify state in the sqlite3ExprCanBeNull() routine. FossilOrigin-Name: be19b84c9f3fe127165809908add148dbe9a827a55608b0490de7e69b7f7f191 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a10a16749a..d88f42963c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sstray\scomment\sin\sthe\sJSON\scode. -D 2023-12-19T00:07:38.941 +C Extra\sALWAYS()\smacros\sto\sverify\sstate\sin\sthe\ssqlite3ExprCanBeNull()\sroutine. +D 2023-12-19T11:57:29.394 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -686,7 +686,7 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c 8f8ed8a5977a54c678755e43aabb3bbe29984025d086128a93e52675987c34eb +F src/expr.c c21c6d1ba6e73b65ad7dd08336956c631b35a3c44935a8b4f26a766493642fa4 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a6a1367b0bf364b1a2e20e153c5f4a578624b8846f9ec0b7c9c3cba0ea2ec346 -R 8cbf9e55e2fcc826ff07211e47ebf076 +P 6618bdf0679405b43911ea8cd94050b12a5dc469f3dfe4759ee3ff850a55229e +R faff5376645090f5364e51a25f92344d U drh -Z 1332cee5da58f474b98f2121b7eff333 +Z 32c2d52f17e83189a0a2418901ffc969 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0ba65215cd..3abdc1543e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6618bdf0679405b43911ea8cd94050b12a5dc469f3dfe4759ee3ff850a55229e \ No newline at end of file +be19b84c9f3fe127165809908add148dbe9a827a55608b0490de7e69b7f7f191 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 756fcc5699..ebe0721cad 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2743,6 +2743,8 @@ int sqlite3ExprCanBeNull(const Expr *p){ p->y.pTab==0 || /* Reference to column of index on expression */ (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ + && ALWAYS(p->iColumn>=0) + && ALWAYS(p->iColumny.pTab->nCol) && p->y.pTab->aCol[p->iColumn].notNull==0); default: return 1; From 3a0e82611ae54859402ca544c6952119b9e48134 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 12:23:11 +0000 Subject: [PATCH 55/80] Always make the sqlite_dbdata virtual table available in the CLI. FossilOrigin-Name: e5fd3b32ad87586a7413570e568c9c1859a37a4f836cca074126471b125fb682 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d88f42963c..97e1dfe1c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extra\sALWAYS()\smacros\sto\sverify\sstate\sin\sthe\ssqlite3ExprCanBeNull()\sroutine. -D 2023-12-19T11:57:29.394 +C Always\smake\sthe\ssqlite_dbdata\svirtual\stable\savailable\sin\sthe\sCLI. +D 2023-12-19T12:23:11.793 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -737,7 +737,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c da0a596a645cd7a8a9fd030e5126600b5639cc63f8ead18c1b67ff5a8a9b6a7f F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c f1a81ff4f8e9e76c224e2ab3a4baa799add0db22158c7fcede65d8cc4a6fa2da -F src/shell.c.in 0df801a0445af3fc55c7330bcd8396ca5433154bb0f728edc8be2d6f27764361 +F src/shell.c.in 7538a5d8d15fb7c67418fc4901275847f0a7af9e7d504df410ab0d8e80c50a4c F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6618bdf0679405b43911ea8cd94050b12a5dc469f3dfe4759ee3ff850a55229e -R faff5376645090f5364e51a25f92344d +P be19b84c9f3fe127165809908add148dbe9a827a55608b0490de7e69b7f7f191 +R 212fb96bb5d7fedd9358acaa914f43a5 U drh -Z 32c2d52f17e83189a0a2418901ffc969 +Z 8226b721ee57239e8a25af6ad532c571 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3abdc1543e..9e4fda1e57 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be19b84c9f3fe127165809908add148dbe9a827a55608b0490de7e69b7f7f191 \ No newline at end of file +e5fd3b32ad87586a7413570e568c9c1859a37a4f836cca074126471b125fb682 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 26b44e05a9..2a075f90ce 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5356,6 +5356,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_regexp_init(p->db, 0, 0); sqlite3_ieee_init(p->db, 0, 0); sqlite3_series_init(p->db, 0, 0); + sqlite3_dbdata_init(p->db, 0, 0); #ifndef SQLITE_SHELL_FIDDLE sqlite3_fileio_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); From 60c435dc4ba0dd0971008fb5d72287ed60c30948 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 12:49:35 +0000 Subject: [PATCH 56/80] When unable to resolve an identifier, change the Expr node into TK_NULL rather than TK_COLUMN, to prevent any downstream misuse of the non-existent column. dbsqlfuzz 71869261db80a95e4733afa10ff5724bf3c78592. FossilOrigin-Name: d2e6117e4f97ab98b01deb5fcad5520f8181d00bed8d904d34963c01d73df857 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/resolve.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 97e1dfe1c5..e4c1331ea0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\smake\sthe\ssqlite_dbdata\svirtual\stable\savailable\sin\sthe\sCLI. -D 2023-12-19T12:23:11.793 +C When\sunable\sto\sresolve\san\sidentifier,\schange\sthe\sExpr\snode\sinto\sTK_NULL\nrather\sthan\sTK_COLUMN,\sto\sprevent\sany\sdownstream\smisuse\sof\sthe\snon-existent\ncolumn.\s\sdbsqlfuzz\s71869261db80a95e4733afa10ff5724bf3c78592. +D 2023-12-19T12:49:35.296 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -734,7 +734,7 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c da0a596a645cd7a8a9fd030e5126600b5639cc63f8ead18c1b67ff5a8a9b6a7f +F src/resolve.c e25f51a473a5f30a0d978e4df2aaa98aeec84eac29ecae1ad4708a6c3e669345 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c f1a81ff4f8e9e76c224e2ab3a4baa799add0db22158c7fcede65d8cc4a6fa2da F src/shell.c.in 7538a5d8d15fb7c67418fc4901275847f0a7af9e7d504df410ab0d8e80c50a4c @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P be19b84c9f3fe127165809908add148dbe9a827a55608b0490de7e69b7f7f191 -R 212fb96bb5d7fedd9358acaa914f43a5 +P e5fd3b32ad87586a7413570e568c9c1859a37a4f836cca074126471b125fb682 +R 48b9c71249fc940d37d3cf9881db1921 U drh -Z 8226b721ee57239e8a25af6ad532c571 +Z 1b1e95f8d5ee358910316b6cde4b39c4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9e4fda1e57..8c09dd33fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e5fd3b32ad87586a7413570e568c9c1859a37a4f836cca074126471b125fb682 \ No newline at end of file +d2e6117e4f97ab98b01deb5fcad5520f8181d00bed8d904d34963c01d73df857 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 43592f8abc..b4f03fe7e6 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -759,6 +759,7 @@ static int lookupName( sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); pParse->checkSchema = 1; pTopNC->nNcErr++; + eNewExprOp = TK_NULL; } assert( pFJMatch==0 ); From 9c36095367d40146167b1edf328e791537dca77c Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 13:00:35 +0000 Subject: [PATCH 57/80] Test case for the previous check-in. FossilOrigin-Name: df5a07e1a5122e08c2fa6076ac08adb2820f997ee11dd88b84863666899dfb57 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/unionall.test | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e4c1331ea0..8306ef6191 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sunable\sto\sresolve\san\sidentifier,\schange\sthe\sExpr\snode\sinto\sTK_NULL\nrather\sthan\sTK_COLUMN,\sto\sprevent\sany\sdownstream\smisuse\sof\sthe\snon-existent\ncolumn.\s\sdbsqlfuzz\s71869261db80a95e4733afa10ff5724bf3c78592. -D 2023-12-19T12:49:35.296 +C Test\scase\sfor\sthe\sprevious\scheck-in. +D 2023-12-19T13:00:35.897 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1864,7 +1864,7 @@ F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 1aeb81976841a91eef292723649b5c4fe3bc3cac F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unhex.test b7f1b806207cb77fa31c3e434fe92fba524464e3e9356809bfcc28f15af1a8b7 -F test/unionall.test eb9afa030897af75fd2f0dd28354ef63c8a5897b6c76aa1f15acae61a12eabcf +F test/unionall.test 6a1cf76f9a35dca9a2d8fbb0d9c709d63338ca3208e6b58972ddff5f4267df89 F test/unionall2.test 71e8fa08d5699d50dc9f9dc0c9799c2e7a6bb7931a330d369307a4df7f157fa1 F test/unionallfault.test 652bfbb630e6c43135965dc1e8f0a9a791da83aec885d626a632fe1909c56f73 F test/unionvtab.test e1704ab1b4c1bb3ffc9da4681f8e85a0b909fd80b937984fc94b27415ac8e5a4 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e5fd3b32ad87586a7413570e568c9c1859a37a4f836cca074126471b125fb682 -R 48b9c71249fc940d37d3cf9881db1921 +P d2e6117e4f97ab98b01deb5fcad5520f8181d00bed8d904d34963c01d73df857 +R 90dde9152d7c6b35edb0cb5f83423d69 U drh -Z 1b1e95f8d5ee358910316b6cde4b39c4 +Z 69483924e6d2b900a24835137337ec8c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8c09dd33fc..db1c072b4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2e6117e4f97ab98b01deb5fcad5520f8181d00bed8d904d34963c01d73df857 \ No newline at end of file +df5a07e1a5122e08c2fa6076ac08adb2820f997ee11dd88b84863666899dfb57 \ No newline at end of file diff --git a/test/unionall.test b/test/unionall.test index 32fc76543a..1c580d2912 100644 --- a/test/unionall.test +++ b/test/unionall.test @@ -346,6 +346,11 @@ do_execsql_test 5.10 { do_execsql_test 5.20 { SELECT *, '+' FROM t1 LEFT JOIN t3 ON (a NOT IN(SELECT v FROM t1 LEFT JOIN t2 ON (a=k))=k); } {0 {} {} {} + 1 one {} {} + 2 two {} {} + 5 five {} {} + 3 three {} {} + 6 six {} {} +} +do_catchsql_test 5.30 { + SELECT * FROM (t1 NATURAL JOIN pragma_table_xinfo('t1_a') NATURAL JOIN t3) t1 + NATURAL JOIN t2 NATURAL JOIN t3 + WHERE rowid ISNULL>0 AND 0%y; +} {1 {no such column: rowid}} reset_db do_execsql_test 6.0 { From 3e2ec83e0bad02934b2c024d123f1a293c214a4e Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 13:45:05 +0000 Subject: [PATCH 58/80] Ignore COLLATE operators when determining whether the result of a subexpression should be shallow-copied or deep-copied. FossilOrigin-Name: 34ae36a45e814bed7c8340412c7ef3fc849b82357656d0eb5f0f805e59d846d0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8306ef6191..8878fa687b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scase\sfor\sthe\sprevious\scheck-in. -D 2023-12-19T13:00:35.897 +C Ignore\sCOLLATE\soperators\swhen\sdetermining\swhether\sthe\sresult\sof\sa\ssubexpression\nshould\sbe\sshallow-copied\sor\sdeep-copied. +D 2023-12-19T13:45:05.259 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -686,7 +686,7 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c c21c6d1ba6e73b65ad7dd08336956c631b35a3c44935a8b4f26a766493642fa4 +F src/expr.c 74c96889f705c83cb7f19d1dd25d937d28708ba079e9b0f8ee6440c7b272b228 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d2e6117e4f97ab98b01deb5fcad5520f8181d00bed8d904d34963c01d73df857 -R 90dde9152d7c6b35edb0cb5f83423d69 +P df5a07e1a5122e08c2fa6076ac08adb2820f997ee11dd88b84863666899dfb57 +R 88cb52429199464661f76b121d4935cc U drh -Z 69483924e6d2b900a24835137337ec8c +Z 118a5efad123e5c170192e2943b0be05 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index db1c072b4d..eb52008aa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df5a07e1a5122e08c2fa6076ac08adb2820f997ee11dd88b84863666899dfb57 \ No newline at end of file +34ae36a45e814bed7c8340412c7ef3fc849b82357656d0eb5f0f805e59d846d0 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index ebe0721cad..fd462c1c54 100644 --- a/src/expr.c +++ b/src/expr.c @@ -5326,8 +5326,10 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); if( inReg!=target ){ u8 op; - if( ALWAYS(pExpr) - && (ExprHasProperty(pExpr,EP_Subquery) || pExpr->op==TK_REGISTER) + Expr *pX = sqlite3ExprSkipCollateAndLikely(pExpr); + testcase( pX!=pExpr ); + if( ALWAYS(pX) + && (ExprHasProperty(pX,EP_Subquery) || pX->op==TK_REGISTER) ){ op = OP_Copy; }else{ From 09259aff6c4ad61b3bbad8228c0a398fa0f92d08 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 14:53:39 +0000 Subject: [PATCH 59/80] Add ALWAYS() and NEVER() on branches made unreachable by recent changes. FossilOrigin-Name: c50e6c2ace49d0928b05cbfd877c621e9a0f77dc4e056ccb1dbe5cf118a00d00 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8878fa687b..953c6c496e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ignore\sCOLLATE\soperators\swhen\sdetermining\swhether\sthe\sresult\sof\sa\ssubexpression\nshould\sbe\sshallow-copied\sor\sdeep-copied. -D 2023-12-19T13:45:05.259 +C Add\sALWAYS()\sand\sNEVER()\son\sbranches\smade\sunreachable\sby\srecent\schanges. +D 2023-12-19T14:53:39.236 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -686,7 +686,7 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c 74c96889f705c83cb7f19d1dd25d937d28708ba079e9b0f8ee6440c7b272b228 +F src/expr.c 49e629c339d3d4f287006325e60878526da0eeb1eb945ff22c8ce1872f6991fe F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P df5a07e1a5122e08c2fa6076ac08adb2820f997ee11dd88b84863666899dfb57 -R 88cb52429199464661f76b121d4935cc +P 34ae36a45e814bed7c8340412c7ef3fc849b82357656d0eb5f0f805e59d846d0 +R 230ae607a1ac4af9df29a9d81c13bbb8 U drh -Z 118a5efad123e5c170192e2943b0be05 +Z 438b65489898c6edaccdbf5c80ee427e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index eb52008aa9..4d47d9517a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34ae36a45e814bed7c8340412c7ef3fc849b82357656d0eb5f0f805e59d846d0 \ No newline at end of file +c50e6c2ace49d0928b05cbfd877c621e9a0f77dc4e056ccb1dbe5cf118a00d00 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index fd462c1c54..0683609526 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2740,7 +2740,7 @@ int sqlite3ExprCanBeNull(const Expr *p){ case TK_COLUMN: assert( ExprUseYTab(p) ); return ExprHasProperty(p, EP_CanBeNull) || - p->y.pTab==0 || /* Reference to column of index on expression */ + NEVER(p->y.pTab==0) || /* Reference to column of index on expr */ (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ && ALWAYS(p->iColumn>=0) @@ -6477,7 +6477,7 @@ static int exprRefToSrcList(Walker *pWalker, Expr *pExpr){ int i; struct RefSrcList *p = pWalker->u.pRefSrcList; SrcList *pSrc = p->pRef; - int nSrc = pSrc ? pSrc->nSrc : 0; + int nSrc = ALWAYS(pSrc) ? pSrc->nSrc : 0; for(i=0; iiTable==pSrc->a[i].iCursor ){ pWalker->eCode |= 1; From 611b9d3efd03c771811175f7a8d25bd7a3007ffd Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 14:54:52 +0000 Subject: [PATCH 60/80] More precise computation of the size of data structures in the query planner. Response to [forum:/forumpost/7d8685d49d|Forum post 7d8685d49d]. FossilOrigin-Name: 0c8d88e41167ea92341dd1129be01b596a73f46bdcd5b0dd931441a979c013d0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 953c6c496e..a7282ce5e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sALWAYS()\sand\sNEVER()\son\sbranches\smade\sunreachable\sby\srecent\schanges. -D 2023-12-19T14:53:39.236 +C More\sprecise\scomputation\sof\sthe\ssize\sof\sdata\sstructures\sin\sthe\squery\splanner.\nResponse\sto\s[forum:/forumpost/7d8685d49d|Forum\spost\s7d8685d49d]. +D 2023-12-19T14:54:52.797 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -821,7 +821,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c dad4b472b8dc886c0962409e70978568c06b5108bcf3dcd4938a9305a3540440 +F src/where.c e80c22f033edc5bdaf992e32bb740831d90a19e8bc3e68523a27add2478c1f65 F src/whereInt.h 4b38c5889514e3aead3f27d0ee9a26e47c3f150efc59e2a8b4e3bc8835e4d7a1 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 34ae36a45e814bed7c8340412c7ef3fc849b82357656d0eb5f0f805e59d846d0 -R 230ae607a1ac4af9df29a9d81c13bbb8 +P c50e6c2ace49d0928b05cbfd877c621e9a0f77dc4e056ccb1dbe5cf118a00d00 +R 5113ea05f125ae14859665eea256b645 U drh -Z 438b65489898c6edaccdbf5c80ee427e +Z f458582e6d386df7f8168eaec2603f07 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4d47d9517a..82c46f6dfd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c50e6c2ace49d0928b05cbfd877c621e9a0f77dc4e056ccb1dbe5cf118a00d00 \ No newline at end of file +0c8d88e41167ea92341dd1129be01b596a73f46bdcd5b0dd931441a979c013d0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c98f92d5a8..15f206a099 100644 --- a/src/where.c +++ b/src/where.c @@ -6038,7 +6038,10 @@ WhereInfo *sqlite3WhereBegin( ** field (type Bitmask) it must be aligned on an 8-byte boundary on ** some architectures. Hence the ROUND8() below. */ - nByteWInfo = ROUND8P(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); + nByteWInfo = ROUND8P(sizeof(WhereInfo)); + if( nTabList>1 ){ + nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel)); + } pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); From 4449a1b66ded7449aa9c82021a7e0fd5ba046153 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 15:06:40 +0000 Subject: [PATCH 61/80] Fix harmless compiler warning in the randomjson.c extension. FossilOrigin-Name: debe7060b16669ada7304ffb9bf7616c8fa30bd286d8be871ed17fd6d64a3d4c --- ext/misc/randomjson.c | 18 +++++++++--------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ext/misc/randomjson.c b/ext/misc/randomjson.c index 5036fa6047..cacc4feb68 100644 --- a/ext/misc/randomjson.c +++ b/ext/misc/randomjson.c @@ -157,27 +157,27 @@ static void jsonExpand( } n = strlen(z); if( (zX = strstr(z,"XX"))!=0 ){ - unsigned int r = prngInt(p); - if( (r&0xff)==((r>>8)&0xff) ) r += 0x100; - while( (r&0xff)==((r>>16)&0xff) || ((r>>8)&0xff)==((r>>16)&0xff) ){ - r += 0x10000; + unsigned int y = prngInt(p); + if( (y&0xff)==((y>>8)&0xff) ) y += 0x100; + while( (y&0xff)==((y>>16)&0xff) || ((y>>8)&0xff)==((y>>16)&0xff) ){ + y += 0x10000; } memcpy(zBuf, z, n+1); z = zBuf; zX = strstr(z,"XX"); while( zX!=0 ){ - zX[0] = "0123456789abcdef"[r%16]; r /= 16; - zX[1] = "0123456789abcdef"[r%16]; r /= 16; + zX[0] = "0123456789abcdef"[y%16]; y /= 16; + zX[1] = "0123456789abcdef"[y%16]; y /= 16; zX = strstr(zX, "XX"); } }else if( (zX = strstr(z,"DD"))!=0 ){ - unsigned int r = prngInt(p); + unsigned int y = prngInt(p); memcpy(zBuf, z, n+1); z = zBuf; zX = strstr(z,"DD"); while( zX!=0 ){ - zX[0] = "0123456789"[r%10]; r /= 10; - zX[1] = "0123456789"[r%10]; r /= 10; + zX[0] = "0123456789"[y%10]; y /= 10; + zX[1] = "0123456789"[y%10]; y /= 10; zX = strstr(zX, "DD"); } } diff --git a/manifest b/manifest index a7282ce5e3..eae227c887 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\sprecise\scomputation\sof\sthe\ssize\sof\sdata\sstructures\sin\sthe\squery\splanner.\nResponse\sto\s[forum:/forumpost/7d8685d49d|Forum\spost\s7d8685d49d]. -D 2023-12-19T14:54:52.797 +C Fix\sharmless\scompiler\swarning\sin\sthe\srandomjson.c\sextension. +D 2023-12-19T15:06:40.294 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -394,7 +394,7 @@ F ext/misc/pcachetrace.c f4227ce03fb16aa8d6f321b72dd051097419d7a028a9853af048bee F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c -F ext/misc/randomjson.c 182cabbeb294f56f68ba2fd0879790e6d4889bb50b6a532604cc6f360d6dd516 +F ext/misc/randomjson.c ef835fc64289e76ac4873b85fe12f9463a036168d7683cf2b773e36e6262c4ed F ext/misc/regexp.c 4bdd0045912f81c84908bd535ec5ad3b1c8540b4287c70ab84070963624047db F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c50e6c2ace49d0928b05cbfd877c621e9a0f77dc4e056ccb1dbe5cf118a00d00 -R 5113ea05f125ae14859665eea256b645 +P 0c8d88e41167ea92341dd1129be01b596a73f46bdcd5b0dd931441a979c013d0 +R c1042677b8772f9b1fe3fffc7f447d0a U drh -Z f458582e6d386df7f8168eaec2603f07 +Z b0164381c9418cb1da631e7b0aaa5ec5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 82c46f6dfd..5c1715f29a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c8d88e41167ea92341dd1129be01b596a73f46bdcd5b0dd931441a979c013d0 \ No newline at end of file +debe7060b16669ada7304ffb9bf7616c8fa30bd286d8be871ed17fd6d64a3d4c \ No newline at end of file From 7c1033b0103b20aca5d1be1bf98f828843bf631b Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 15:10:25 +0000 Subject: [PATCH 62/80] On second thought, we don't really need sqlite_dbdata accessible to the CLI. FossilOrigin-Name: 36fe6a61ef8fb393281a5e15119d716521219c7b971fbfd63bdea07d27a78ac9 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/shell.c.in | 1 - 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index eae227c887..80341fe69e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sin\sthe\srandomjson.c\sextension. -D 2023-12-19T15:06:40.294 +C On\ssecond\sthought,\swe\sdon't\sreally\sneed\ssqlite_dbdata\saccessible\sto\sthe\sCLI. +D 2023-12-19T15:10:25.467 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -737,7 +737,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c e25f51a473a5f30a0d978e4df2aaa98aeec84eac29ecae1ad4708a6c3e669345 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c f1a81ff4f8e9e76c224e2ab3a4baa799add0db22158c7fcede65d8cc4a6fa2da -F src/shell.c.in 7538a5d8d15fb7c67418fc4901275847f0a7af9e7d504df410ab0d8e80c50a4c +F src/shell.c.in 0df801a0445af3fc55c7330bcd8396ca5433154bb0f728edc8be2d6f27764361 F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2155,8 +2155,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0c8d88e41167ea92341dd1129be01b596a73f46bdcd5b0dd931441a979c013d0 -R c1042677b8772f9b1fe3fffc7f447d0a +P debe7060b16669ada7304ffb9bf7616c8fa30bd286d8be871ed17fd6d64a3d4c +Q -e5fd3b32ad87586a7413570e568c9c1859a37a4f836cca074126471b125fb682 +R 2046736d122f4a4df664dffbbe842afc U drh -Z b0164381c9418cb1da631e7b0aaa5ec5 +Z a020f3de47e20cdfc71436e3b7fdedad # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5c1715f29a..73f14492b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -debe7060b16669ada7304ffb9bf7616c8fa30bd286d8be871ed17fd6d64a3d4c \ No newline at end of file +36fe6a61ef8fb393281a5e15119d716521219c7b971fbfd63bdea07d27a78ac9 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 2a075f90ce..26b44e05a9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5356,7 +5356,6 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_regexp_init(p->db, 0, 0); sqlite3_ieee_init(p->db, 0, 0); sqlite3_series_init(p->db, 0, 0); - sqlite3_dbdata_init(p->db, 0, 0); #ifndef SQLITE_SHELL_FIDDLE sqlite3_fileio_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); From c52aebb443751e01a8800bb493dd62bb435fd98b Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 15:51:14 +0000 Subject: [PATCH 63/80] Remove redundant conditional from sqlite3ExprCanBeNull(). FossilOrigin-Name: 257f96a2d22c605885fa66220c28cf7dc5941c330bccee3f132b9e7b70d89d30 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/expr.c | 1 - 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 80341fe69e..5fcfb15ae3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C On\ssecond\sthought,\swe\sdon't\sreally\sneed\ssqlite_dbdata\saccessible\sto\sthe\sCLI. -D 2023-12-19T15:10:25.467 +C Remove\sredundant\sconditional\sfrom\ssqlite3ExprCanBeNull(). +D 2023-12-19T15:51:14.357 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -686,7 +686,7 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c 49e629c339d3d4f287006325e60878526da0eeb1eb945ff22c8ce1872f6991fe +F src/expr.c 125e6966297727b5d647d73893873b5d0730377f73208dc70e39dea9aa29f366 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 @@ -2155,9 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P debe7060b16669ada7304ffb9bf7616c8fa30bd286d8be871ed17fd6d64a3d4c -Q -e5fd3b32ad87586a7413570e568c9c1859a37a4f836cca074126471b125fb682 -R 2046736d122f4a4df664dffbbe842afc +P 36fe6a61ef8fb393281a5e15119d716521219c7b971fbfd63bdea07d27a78ac9 +R 8d141472bcc17284a46ea446e12c41f3 U drh -Z a020f3de47e20cdfc71436e3b7fdedad +Z 7720e1377fde94e934586312bc65b10d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 73f14492b9..5c345ddc6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -36fe6a61ef8fb393281a5e15119d716521219c7b971fbfd63bdea07d27a78ac9 \ No newline at end of file +257f96a2d22c605885fa66220c28cf7dc5941c330bccee3f132b9e7b70d89d30 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 0683609526..11d052feea 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2743,7 +2743,6 @@ int sqlite3ExprCanBeNull(const Expr *p){ NEVER(p->y.pTab==0) || /* Reference to column of index on expr */ (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ - && ALWAYS(p->iColumn>=0) && ALWAYS(p->iColumny.pTab->nCol) && p->y.pTab->aCol[p->iColumn].notNull==0); default: From 3262ca83a7956582c6eded431e7d5374a63f8bbe Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 19 Dec 2023 21:39:58 +0000 Subject: [PATCH 64/80] In JSON - minor code cleanup and refactoring with a small size reduction and performance increase. FossilOrigin-Name: 215fabda38daecdbd38b1eca5a6aafbc61b6a36a8303f1d7164d5a1138e63134 --- manifest | 12 +++++------ manifest.uuid | 2 +- src/json.c | 58 +++++++++++++++++++++++++-------------------------- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 5fcfb15ae3..c659725846 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sredundant\sconditional\sfrom\ssqlite3ExprCanBeNull(). -D 2023-12-19T15:51:14.357 +C In\sJSON\s-\sminor\scode\scleanup\sand\srefactoring\swith\sa\ssmall\ssize\sreduction\nand\sperformance\sincrease. +D 2023-12-19T21:39:58.157 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 661b6da631185152c635161d52c274255c8de3f5769570bbbde0311cd9ad7460 +F src/json.c c6fae579d681d5ab43387c8667cc6ef0c0f8769fd881fde3af6260e837cae762 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 36fe6a61ef8fb393281a5e15119d716521219c7b971fbfd63bdea07d27a78ac9 -R 8d141472bcc17284a46ea446e12c41f3 +P 257f96a2d22c605885fa66220c28cf7dc5941c330bccee3f132b9e7b70d89d30 +R 1e43c523c137344cae8cdc90cc8b1706 U drh -Z 7720e1377fde94e934586312bc65b10d +Z 35aaca72ca93a246000bef5de54a54bb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5c345ddc6f..fdf94f914d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -257f96a2d22c605885fa66220c28cf7dc5941c330bccee3f132b9e7b70d89d30 \ No newline at end of file +215fabda38daecdbd38b1eca5a6aafbc61b6a36a8303f1d7164d5a1138e63134 \ No newline at end of file diff --git a/src/json.c b/src/json.c index d0e7b3fc6b..8fae3d1140 100644 --- a/src/json.c +++ b/src/json.c @@ -20,9 +20,9 @@ ** All generated JSON text still conforms strictly to RFC-8259, but text ** with JSON-5 extensions is accepted as input. ** -** Beginning with version 3.45.0 (pending), these routines also accept -** BLOB values that have JSON encoded using a binary representation we -** call JSONB. The name JSONB comes from PostgreSQL, however the on-disk +** Beginning with version 3.45.0 (circa 2024-01-01), these routines also +** accept BLOB values that have JSON encoded using a binary representation +** called "JSONB". The name JSONB comes from PostgreSQL, however the on-disk ** format SQLite JSONB is completely different and incompatible with ** PostgreSQL JSONB. ** @@ -220,6 +220,8 @@ typedef struct JsonParse JsonParse; ** * The aBlob[] array must be owned by the JsonParse object. In other ** words, nBlobAlloc must be non-zero. ** +** * eEdit and delta must be zero. +** ** * zJson must be an RCStr. In other words bJsonIsRCStr must be true. */ struct JsonCache { @@ -279,8 +281,8 @@ struct JsonString { ** json_replace() or json_patch() or similar). ** ** 4. New JSON text is generated from the aBlob[] for output. This step -** is skipped the function is one of the jsonb_* functions that returns -** JSONB instead of text JSON. +** is skipped if the function is one of the jsonb_* functions that +** returns JSONB instead of text JSON. */ struct JsonParse { u8 *aBlob; /* JSONB representation of JSON value */ @@ -288,14 +290,14 @@ struct JsonParse { u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ char *zJson; /* Json text used for parsing */ int nJson; /* Length of the zJson string in bytes */ + u32 nJPRef; /* Number of references to this object */ + u32 iErr; /* Error location in zJson[] */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ u8 bJsonIsRCStr; /* True if zJson is an RCStr */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 bReadOnly; /* Do not modify. */ - u32 nJPRef; /* Number of references to this object */ - u32 iErr; /* Error location in zJson[] */ /* Search and edit information. See jsonLookupStep() */ u8 eEdit; /* Edit operation to apply */ int delta; /* Size change due to the edit */ @@ -334,7 +336,7 @@ struct JsonParse { **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); -static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); +static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static void jsonParseFree(JsonParse*); @@ -709,7 +711,7 @@ static void jsonAppendSqlValue( memset(&px, 0, sizeof(px)); px.aBlob = (u8*)sqlite3_value_blob(pValue); px.nBlob = sqlite3_value_bytes(pValue); - jsonXlateBlobToText(&px, 0, p); + jsonTranslateBlobToText(&px, 0, p); }else if( p->eErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); p->eErr = JSTRING_ERR; @@ -1226,15 +1228,11 @@ static int jsonBlobChangePayloadSize( */ static int jsonIs4HexB(const char *z, int *pOp){ if( z[0]!='u' ) return 0; - if( !sqlite3Isxdigit(z[1]) ) return 0; - if( !sqlite3Isxdigit(z[2]) ) return 0; - if( !sqlite3Isxdigit(z[3]) ) return 0; - if( !sqlite3Isxdigit(z[4]) ) return 0; + if( !jsonIs4Hex(&z[1]) ) return 0; *pOp = JSONB_TEXTJ; return 1; } - /* ** Check a single element of the JSONB in pParse for validity. ** @@ -1451,7 +1449,7 @@ static u32 jsonbValidityCheck( ** -4 ',' seen / the index in zJson[] of the seen character ** -5 ':' seen / */ -static int jsonXlateTextToBlob(JsonParse *pParse, u32 i){ +static int jsonTranslateTextToBlob(JsonParse *pParse, u32 i){ char c; u32 j; u32 iThis, iStart; @@ -1471,7 +1469,7 @@ json_parse_restart: iStart = pParse->nBlob; for(j=i+1;;j++){ u32 iBlob = pParse->nBlob; - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x<=0 ){ int op; if( x==(-2) ){ @@ -1517,7 +1515,7 @@ json_parse_restart: goto parse_object_value; } } - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x!=(-5) ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1525,7 +1523,7 @@ json_parse_restart: j = pParse->iErr+1; } parse_object_value: - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x<=0 ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1544,7 +1542,7 @@ json_parse_restart: break; } } - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1572,7 +1570,7 @@ json_parse_restart: return -1; } for(j=i+1;;j++){ - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; @@ -1596,7 +1594,7 @@ json_parse_restart: break; } } - x = jsonXlateTextToBlob(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1919,7 +1917,7 @@ static int jsonConvertTextToBlob( ){ int i; const char *zJson = pParse->zJson; - i = jsonXlateTextToBlob(pParse, 0); + i = jsonTranslateTextToBlob(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ #ifdef SQLITE_DEBUG @@ -1964,7 +1962,7 @@ static void jsonReturnStringAsBlob(JsonString *pStr){ jsonStringTerminate(pStr); px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; - (void)jsonXlateTextToBlob(&px, 0); + (void)jsonTranslateTextToBlob(&px, 0); if( px.oom ){ sqlite3_free(px.aBlob); sqlite3_result_error_nomem(pStr->pCtx); @@ -2052,7 +2050,7 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ ** ** The pOut->eErr JSTRING_OOM flag is set on a OOM. */ -static u32 jsonXlateBlobToText( +static u32 jsonTranslateBlobToText( const JsonParse *pParse, /* the complete parse of the JSON */ u32 i, /* Start rendering at this index */ JsonString *pOut /* Write JSON here */ @@ -2223,7 +2221,7 @@ static u32 jsonXlateBlobToText( j = i+n; iEnd = j+sz; while( j0 ) pOut->nUsed--; @@ -2236,7 +2234,7 @@ static u32 jsonXlateBlobToText( j = i+n; iEnd = j+sz; while( jeErr |= JSTRING_MALFORMED; @@ -2818,7 +2816,7 @@ static void jsonReturnTextJsonFromBlob( x.aBlob = (u8*)aBlob; x.nBlob = nBlob; jsonStringInit(&s, ctx); - jsonXlateBlobToText(&x, 0, &s); + jsonTranslateBlobToText(&x, 0, &s); jsonReturnString(&s, 0, 0); } @@ -3320,7 +3318,7 @@ static void jsonReturnParse( JsonString s; jsonStringInit(&s, ctx); p->delta = 0; - jsonXlateBlobToText(p, 0, &s); + jsonTranslateBlobToText(p, 0, &s); jsonReturnString(&s, p, ctx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -3648,7 +3646,7 @@ static void jsonExtractFunc( if( argc==2 ){ if( flags & JSON_JSON ){ jsonStringInit(&jx, ctx); - jsonXlateBlobToText(p, j, &jx); + jsonTranslateBlobToText(p, j, &jx); jsonReturnString(&jx, 0, 0); jsonStringReset(&jx); assert( (flags & JSON_BLOB)==0 ); @@ -3663,7 +3661,7 @@ static void jsonExtractFunc( } }else{ jsonAppendSeparator(&jx); - jsonXlateBlobToText(p, j, &jx); + jsonTranslateBlobToText(p, j, &jx); } }else if( j==JSON_LOOKUP_NOTFOUND ){ if( argc==2 ){ From 95cf95841c81a777a9268a1a375b30b6a8466628 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 20 Dec 2023 11:34:17 +0000 Subject: [PATCH 65/80] Avoid harmless integer overflow in pager status statistics gathering. Response to [forum:/forumpost/7f4cdf23f9|forum post 7f4cdf23f9]. FossilOrigin-Name: 206d8c650d937bc700946c40a82a62ea6bc4a80e5f3fb42d0ae2968de25f0644 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 10 +++++----- src/pager.h | 2 +- src/status.c | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c659725846..9bc0240ebc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sJSON\s-\sminor\scode\scleanup\sand\srefactoring\swith\sa\ssmall\ssize\sreduction\nand\sperformance\sincrease. -D 2023-12-19T21:39:58.157 +C Avoid\sharmless\sinteger\soverflow\sin\spager\sstatus\sstatistics\sgathering.\nResponse\sto\s[forum:/forumpost/7f4cdf23f9|forum\spost\s7f4cdf23f9]. +D 2023-12-20T11:34:17.563 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -723,8 +723,8 @@ F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d87210 F src/os_unix.c 97bdcd43315da7aaec9fea2da1ff7c9de458f93dd363e073f2742403a7f2e011 F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 987ab3a2cd9065d62e9955474470ff733445e2357432a67e3d0f5a8f9313e334 -F src/pager.h f4d33fec8052603758792045493423b8871a996da2d0973927b7d36cd6070473 +F src/pager.c dc0cccda12d1675f461ef47090ec6bd7bb7611e77dbf5e0796667120d9b67f37 +F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a F src/parse.y 020d80386eb216ec9520549106353c517d2bbc89be28752ffdca649a9eaf56ec F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 @@ -743,7 +743,7 @@ F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 F src/sqliteInt.h 134457f62bb1d0ff1dd037cc23dd46b1d16efbbfc2211dc2b15c380af731d9ac F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 -F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 +F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c ecbc3c99c0d0c3ed122a913f143026c26d38d57f33e06bb71185dd5c1efe37cd F src/test1.c ac6542cddd1f405e332d869946b977b2ce8b4dc28b9f7cc61df38abe1fe49bc3 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 257f96a2d22c605885fa66220c28cf7dc5941c330bccee3f132b9e7b70d89d30 -R 1e43c523c137344cae8cdc90cc8b1706 +P 215fabda38daecdbd38b1eca5a6aafbc61b6a36a8303f1d7164d5a1138e63134 +R d4f2caa7842856e1d7a2268ee871a371 U drh -Z 35aaca72ca93a246000bef5de54a54bb +Z 1d76cf71ba5b0710fab4dfb939c1e975 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fdf94f914d..95de95a991 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -215fabda38daecdbd38b1eca5a6aafbc61b6a36a8303f1d7164d5a1138e63134 \ No newline at end of file +206d8c650d937bc700946c40a82a62ea6bc4a80e5f3fb42d0ae2968de25f0644 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 4687ab0f19..681aa55836 100644 --- a/src/pager.c +++ b/src/pager.c @@ -688,7 +688,7 @@ struct Pager { char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ - int aStat[4]; /* Total cache hits, misses, writes, spills */ + u32 aStat[4]; /* Total cache hits, misses, writes, spills */ #ifdef SQLITE_TEST int nRead; /* Database pages read */ #endif @@ -6832,11 +6832,11 @@ int *sqlite3PagerStats(Pager *pPager){ a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; - a[6] = pPager->aStat[PAGER_STAT_HIT]; - a[7] = pPager->aStat[PAGER_STAT_MISS]; + a[6] = (int)pPager->aStat[PAGER_STAT_HIT] & 0x7fffffff; + a[7] = (int)pPager->aStat[PAGER_STAT_MISS] & 0x7fffffff; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; - a[10] = pPager->aStat[PAGER_STAT_WRITE]; + a[10] = (int)pPager->aStat[PAGER_STAT_WRITE] & 0x7fffffff; return a; } #endif @@ -6852,7 +6852,7 @@ int *sqlite3PagerStats(Pager *pPager){ ** reset parameter is non-zero, the cache hit or miss count is zeroed before ** returning. */ -void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ +void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, u64 *pnVal){ assert( eStat==SQLITE_DBSTATUS_CACHE_HIT || eStat==SQLITE_DBSTATUS_CACHE_MISS diff --git a/src/pager.h b/src/pager.h index 044c2573e6..7ef9a237ae 100644 --- a/src/pager.h +++ b/src/pager.h @@ -216,7 +216,7 @@ sqlite3_file *sqlite3PagerJrnlFile(Pager*); const char *sqlite3PagerJournalname(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); -void sqlite3PagerCacheStat(Pager *, int, int, int *); +void sqlite3PagerCacheStat(Pager *, int, int, u64*); void sqlite3PagerClearCache(Pager*); int sqlite3SectorSize(sqlite3_file *); diff --git a/src/status.c b/src/status.c index 7840167002..a462c94293 100644 --- a/src/status.c +++ b/src/status.c @@ -362,7 +362,7 @@ int sqlite3_db_status( case SQLITE_DBSTATUS_CACHE_MISS: case SQLITE_DBSTATUS_CACHE_WRITE:{ int i; - int nRet = 0; + u64 nRet = 0; assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); @@ -375,7 +375,7 @@ int sqlite3_db_status( *pHighwater = 0; /* IMP: R-42420-56072 */ /* IMP: R-54100-20147 */ /* IMP: R-29431-39229 */ - *pCurrent = nRet; + *pCurrent = (int)nRet & 0x7fffffff; break; } From 0d7f0e49a405484b8fa7fcad76f55b648ad8574b Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Dec 2023 19:33:41 +0000 Subject: [PATCH 66/80] Fix SQLITE_ENABLE_SETLK_TIMEOUT assert() statements in os_unix.c to avoid reading past the end of the unixShmNode.aMutex[] array. FossilOrigin-Name: 029a05cd2928d43d81e4549cce5388c432e2c9e75e3fa0b2fe6e91021b2fb9ac --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 12 +++++++++--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9bc0240ebc..801a4eda9d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sharmless\sinteger\soverflow\sin\spager\sstatus\sstatistics\sgathering.\nResponse\sto\s[forum:/forumpost/7f4cdf23f9|forum\spost\s7f4cdf23f9]. -D 2023-12-20T11:34:17.563 +C Fix\sSQLITE_ENABLE_SETLK_TIMEOUT\sassert()\sstatements\sin\sos_unix.c\sto\savoid\sreading\spast\sthe\send\sof\sthe\sunixShmNode.aMutex[]\sarray. +D 2023-12-20T19:33:41.679 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -720,7 +720,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c 97bdcd43315da7aaec9fea2da1ff7c9de458f93dd363e073f2742403a7f2e011 +F src/os_unix.c 1672c708df279fca1b6ba619cbb26a88baa7913b21dda95817290d76666a9688 F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c dc0cccda12d1675f461ef47090ec6bd7bb7611e77dbf5e0796667120d9b67f37 @@ -2155,8 +2155,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 215fabda38daecdbd38b1eca5a6aafbc61b6a36a8303f1d7164d5a1138e63134 -R d4f2caa7842856e1d7a2268ee871a371 -U drh -Z 1d76cf71ba5b0710fab4dfb939c1e975 +P 206d8c650d937bc700946c40a82a62ea6bc4a80e5f3fb42d0ae2968de25f0644 +R f91549b5ba9f8490543a9270abff43ad +U dan +Z 03359f67466d165815d902ddaee87b98 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 95de95a991..4428cae8a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -206d8c650d937bc700946c40a82a62ea6bc4a80e5f3fb42d0ae2968de25f0644 \ No newline at end of file +029a05cd2928d43d81e4549cce5388c432e2c9e75e3fa0b2fe6e91021b2fb9ac \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 7362a13206..21bbd97694 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4434,9 +4434,15 @@ static int unixShmSystemLock( pShmNode = pFile->pInode->pShmNode; - /* Assert that the correct mutex or mutexes are held. */ - if( pShmNode->nRef==0 ){ - assert( ofst==UNIX_SHM_DMS && n==1 && unixMutexHeld() ); + /* Assert that the parameters are within expected range and that the + ** correct mutex or mutexes are held. */ + assert( pShmNode->nRef>=0 ); + assert( (ofst==UNIX_SHM_DMS && n==1) + || (ofst>=UNIX_SHM_BASE && ofst+n<=(UNIX_SHM_BASE+SQLITE_SHM_NLOCK)) + ); + if( ofst==UNIX_SHM_DMS ){ + assert( pShmNode->nRef>0 || unixMutexHeld() ); + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); }else{ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT int ii; From c2eff91bd320b0199cd37b0c448466f3262a8f74 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 21 Dec 2023 18:08:05 +0000 Subject: [PATCH 67/80] Add internal core-developer-only documentation of the JSONB format. FossilOrigin-Name: 4d30478863b2a60512010de9ec6e3099bfaf75d4afee20acec536713fe94334d --- doc/jsonb.md | 290 ++++++++++++++++++++++++++++++++++++++++++++++++++ manifest | 15 +-- manifest.uuid | 2 +- src/json.c | 3 + 4 files changed, 302 insertions(+), 8 deletions(-) create mode 100644 doc/jsonb.md diff --git a/doc/jsonb.md b/doc/jsonb.md new file mode 100644 index 0000000000..5beed1631d --- /dev/null +++ b/doc/jsonb.md @@ -0,0 +1,290 @@ +# The JSONB Format + +This document describes SQLite's JSONB binary encoding of +JSON. + +## 1.0 What Is JSONB? + +Beginning with version 3.45.0 (circa 2024-01-01), SQLite supports an +alternative binary encoding of JSON which we call "JSONB". JSONB is +a binary format that stored as a BLOB. + +The advantage of JSONB over ordinary text RFC 8259 JSON is that JSONB +is both slightly smaller (by between 5% and 10% in most cases) and +can be processed in less than half the number of CPU cycles. The built-in +[JSON SQL functions] of SQLite can accept either ordinary text JSON +or the binary JSONB encoding for any of their JSON inputs. + +The "JSONB" name is inspired by [PostgreSQL](https://postgresql.org), but the +on-disk format for SQLite's JSONB is not the same as PostgreSQL's. +The two formats have the same name, but they have wildly different internal +representations and are not in any way binary compatible. + +The central idea behind this JSONB specification is that each element +begins with a header that includes the size and type of that element. +The header takes the place of punctuation such as double-quotes, +curly-brackes, square-brackets, commas, and colons. Since the size +and type of each element is contained in its header, the element can +be read faster since it is no longer necessary to carefully scan forward +looking for the closing delimiter. The payload of JSONB is the same +as for corresponding text JSON. The same payload bytes occur in the +same order. The only real difference between JSONB and ordinary text +JSON is that JSONB includes a binary header on +each element and omits delimiter and separator punctuation. + +### 1.1 Internal Use Only + +The details of the JSONB are not intended to be visible to application +developers. Application developers should look at JSONB as an opaque BLOB +used internally by SQLite. Nevertheless, we want the format to be backwards +compatible across all future versions of SQLite. To that end, the format +is documented by this file in the source tree. But this file should be +used only by SQLite core developers, not by developers of applications +that only use SQLite. + +## 2.0 The Purpose Of This Document + +JSONB is not intended as an external format to be used by +applications. JSONB is designed for internal use by SQLite only. +Programmers do not need to understand the JSONB format in order to +use it effectively. +Applications should access JSONB only through the [JSON SQL functions], +not by looking at individual bytes of the BLOB. + +However, JSONB is intended to be portable and backwards compatible +for all future versions of SQLite. In other words, you should not have +to export and reimport your SQLite database files when you upgrade to +a newer SQLite version. For that reason, the JSONB format needs to +be well-defined. + +This document is therefore similar in purpose to the +[SQLite database file format] document that describes the on-disk +format of an SQLite database file. Applications are not expected +to directly read and write the bits and bytes of SQLite database files. +The SQLite database file format is carefully documented so that it +can be stable and enduring. In the same way, the JSONB representation +of JSON is documented here so that it too can be stable and enduring, +not so that applications can read or writes individual bytes. + +## 3.0 Encoding + +JSONB is a direct translation of the underlying text JSON. The difference +is that JSONB uses a binary encoding that is faster to parse compared to +the detailed syntax of text JSON. + +Each JSON element is encoded as a header and a payload. The header +determines type of element (string, numeric, boolean, null, object, or +array) and the size of the payload. The header can be between 1 and +9 bytes in size. The payload can be any size from zero bytes up to the +maximum allowed BLOB size. + +### 3.1 Payload Size + +The upper four bits of the first byte of the header determine size of the +header and possibly also the size of the payload. +If the upper four bits have a value between 0 and 11, then the header is +exactly one byte in size and the payload size is determined by those +upper four bits. If the upper four bits have a value between 12 and 15, +that means that the total header size is 2, 3, 5, or 9 bytes and the +payload size is unsigned big-endian integer that is contained in the +subsequent bytes. The size integer is the one byte that following the +initial header byte if the upper four bits +are 12, two bytes if the upper bits are 13, four bytes if the upper bits +are 14, and eight bytes if the upper bits are 15. The current design +of SQLite does not support BLOB values larger than 2GiB, so the eight-byte +variant of the payload size integer will never be used by the current code. +The eight-byte payload size integer is included in the specification +to allow for future expansion. + +The header for an element does *not* need to be in its simplest +form. For example, consider the JSON numeric value "`1`". +That element can be encode in five different ways: + + * `0x13 0x31` + * `0xc3 0x01 0x31` + * `0xd3 0x00 0x01 0x31` + * `0xe3 0x00 0x00 0x00 0x01 0x31` + * `0xf3 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x01 0x31` + +The shortest encoding is preferred, of course, and usually happens with +primitive elements such as numbers. However the total size of an array +or object might not be known exactly when the header of the element is +first generated. It is convenient to reserve space for the largest +possible header and then go back and fill in the correct payload size +at the end. This technique can result in array or object headers that +are larger than absolutely necessary. + +### 3.2 Element Type + +The least-significant four bits of the first byte of the header (the first +byte masked against 0x0f) determine element type. The following codes are +used: + +
    +

  1. NULL → +The element is a JSON "null". The payload size for a true JSON NULL must +must be zero. Future versions of SQLite might extend the JSONB format +with elements that have a zero element type but a non-zero size. In that +way, legacy versions of SQLite will interpret the element as a NULL +for backwards compatibility while newer versions will interpret the +element in some other way. + +

  2. TRUE → +The element is a JSON "true". The payload size must be zero for a actual +"true" value. Elements with type 1 and a non-zero payload size are +reserved for future expansion. Legacy implementations that see an element +type of 1 with a non-zero payload size should continue to interpret that +element as "true" for compatibility. + +

  3. FALSE → +The element is a JSON "false". The payload size must be zero for a actual +"false" value. Elements with type 2 and a non-zero payload size are +reserved for future expansion. Legacy implementations that see an element +type of 2 with a non-zero payload size should continue to interpret that +element as "false" for compatibility. + +

  4. INT → +The element is a JSON integer value in the canonical +RFC 8259 format, without extensions. The payload is the ASCII +text representation of that numeric value. + +

  5. INT5 → +The element is a JSON integer value that is not in the +canonical format. The payload is the ASCII +text representation of that numeric value. Because the payload is in a +non-standard format, it will need to be translated when the JSONB is +converted into RFC 8259 text JSON. + +

  6. FLOAT → +The element is a JSON floating-point value in the canonical +RFC 8259 format, without extensions. The payload is the ASCII +text representation of that numeric value. + +

  7. FLOAT5 → +The element is a JSON floating-point value that is not in the +canonical format. The payload is the ASCII +text representation of that numeric value. Because the payload is in a +non-standard format, it will need to be translated when the JSONB is +converted into RFC 8259 text JSON. + +

  8. TEXT → +The element is a JSON string value that does not contain +any escapes nor any characters that need to be escaped for either SQL or +JSON. The payload is the UTF8 text representation of the string value. +The payload does not include string delimiters. + +

  9. TEXTJ → +The element is a JSON string value that contains +RFC 8259 character escapes (such as "\n" or "\u0020"). +Those escapes will need to be translated into actual UTF8 if this element +is [json_extract|extracted] into SQL. +The payload is the UTF8 text representation of the escaped string value. +The payload does not include string delimiters. + +

  10. TEXT5 → +The element is a JSON string value that contains +character escapes, including some character escapes that part of JSON5 +and which are not found in the canonical RFC 8259 spec. +Those escapes will need to be translated into standard JSON prior to +rendering the JSON as text, or into their actual UTF8 characters if this +element is [json_extract|extracted] into SQL. +The payload is the UTF8 text representation of the escaped string value. +The payload does not include string delimiters. + +

  11. TEXTRAW → +The element is a JSON string value that contains +UTF8 characters that need to be escaped if this string is rendered into +standard JSON text. +The payload does not include string delimiters. + +

  12. ARRAY → +The element is a JSON array. The payload contains +JSONB elements that comprise values contained within the array. + +

  13. OBJECT → +The element is a JSON object. The payload contains +pairs of JSONB elements that comprise entries for the JSON object. +The first element in each pair must be a string (types 7 through 10). +The second element of each pair may be any types, including nested +arrays or objects. + +

  14. RESERVED-13 → +Reserved for future expansion. Legacy implements that encounter this +element type should raise an error. + +

  15. RESERVED-14 → +Reserved for future expansion. Legacy implements that encounter this +element type should raise an error. + +

  16. RESERVED-15 → +Reserved for future expansion. Legacy implements that encounter this +element type should raise an error. +

+ +Element types outside the range of 0 to 12 are reserved for future +expansion. The current implement raises an error if see an element type +other than those listed above. However, future versions of SQLite might +use of the three remaining element types to implement indexing or similar +optimizations, to speed up lookup against large JSON arrays and/or objects. + +### 3.3 Design Rationale For Element Types + +A key goal of JSONB is that it should be quick to translate +to and from text JSON and/or be constructed from SQL values. +When converting from text into JSONB, we do not want the +converter subroutine to burn CPU cycles converting elements +values into some standard format which might never be used. +Format conversion is "lazy" - it is deferred until actually +needed. This has implications for the JSONB format design: + + 1. Numeric values are stored as text, not a numbers. The values are + a direct copy of the text JSON values from which they are derived. + + 2. There are multiple element types depending on the details of value + formats. For example, INT is used for pure RFC-8259 integer + literals and INT5 exists for JSON5 extensions such as hexadecimal + notation. FLOAT is used for pure RFC-8259 floating point literals + and FLOAT5 is used for JSON5 extensions. There are four different + representations of strings, depending on where the string came from + and how special characters within the string are escaped. + +A second goal of JSONB is that it should be capable of serving as the +"parse tree" for JSON when a JSON value is being processed by the +various [JSON SQL functions] built into SQLite. Before JSONB was +developed, operations such [json_replace()] and [json_patch()] +and similar worked in three stages: + + + 1. Translate the text JSON into a internal format that is + easier to scan and edit. + 2. Perform the requested operation on the JSON. + 3. Translate the internal format back into text. + +JSONB seeks to serve as the internal format directly - bypassing +the first and third stages of that process. Since most of the CPU +cycles are spent on the first and third stages, that suggests that +JSONB processing will be much faster than text JSON processing. + +So when processing JSONB, only the second stage of the three-stage +process is required. But when processing text JSON, it is still necessary +to do stages one and three. If JSONB is to be used as the internal +binary representation, this is yet another reason to store numeric +values as text. Storing numbers as text minimizes the amount of +conversion work needed for stages one and three. This is also why +there are four different representations of text in JSONB. Different +text representations are used for text coming from different sources +(RFC-8259 JSON, JSON5, or SQL string values) and conversions only +happen if and when they are actually needed. + +### 3.4 Valid JSONB BLOBs + +A valid JSONB BLOB consists of a single JSON element. The element must +exactly fill the BLOB. This one element is often a JSON object or array +and those usually contain additional elements as its payload, but the +element can be a primite value such a string, number, boolean, or null. + +When the built-in JSON functions are attempting to determine if a BLOB +argument is a JSONB or just a random BLOB, they look at the header of +the outer element to see that it is well-formed and that the element +completely fills the BLOB. If these conditions are met, then the BLOB +is accepted as a JSONB value. diff --git a/manifest b/manifest index 801a4eda9d..ef0d0902d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sSQLITE_ENABLE_SETLK_TIMEOUT\sassert()\sstatements\sin\sos_unix.c\sto\savoid\sreading\spast\sthe\send\sof\sthe\sunixShmNode.aMutex[]\sarray. -D 2023-12-20T19:33:41.679 +C Add\sinternal\score-developer-only\sdocumentation\sof\sthe\sJSONB\sformat. +D 2023-12-21T18:08:05.881 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -39,6 +39,7 @@ F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd F doc/compile-for-windows.md 50b27d77be96195c66031a3181cb8684ed822327ea834e07f9c014213e5e3bcf F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f +F doc/jsonb.md 5fab4b8613aa9153fbeb6259297bd4697988af8b3d23900deba588fa7841456b F doc/lemon.html 44a53a1d2b42d7751f7b2f478efb23c978e258d794bfd172442307a755b9fa44 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/testrunner.md 8d36ec692cf4994bb66d84a4645b9afa1ce9d47dc12cbf8d437c5a5fb6ddeedb @@ -696,7 +697,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c c6fae579d681d5ab43387c8667cc6ef0c0f8769fd881fde3af6260e837cae762 +F src/json.c 498cbe3346f216e3119e6d01585c3bc09994612c16459f2fa682a1cec61b1cf7 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2155,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 206d8c650d937bc700946c40a82a62ea6bc4a80e5f3fb42d0ae2968de25f0644 -R f91549b5ba9f8490543a9270abff43ad -U dan -Z 03359f67466d165815d902ddaee87b98 +P 029a05cd2928d43d81e4549cce5388c432e2c9e75e3fa0b2fe6e91021b2fb9ac +R 37c4deae61c278596f4d5133c7cb9761 +U drh +Z eab6e0f06a97a969bebac69bfb9fe946 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4428cae8a4..39fbf7193d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -029a05cd2928d43d81e4549cce5388c432e2c9e75e3fa0b2fe6e91021b2fb9ac \ No newline at end of file +4d30478863b2a60512010de9ec6e3099bfaf75d4afee20acec536713fe94334d \ No newline at end of file diff --git a/src/json.c b/src/json.c index 8fae3d1140..e4f658ca7f 100644 --- a/src/json.c +++ b/src/json.c @@ -113,6 +113,9 @@ ** checks are true, the BLOB is assumed to be JSONB and processing continues. ** Errors are only raised if some other miscoding is discovered during ** processing. +** +** Additional information can be found in the doc/jsonb.md file of the +** canonical SQLite source tree. */ #ifndef SQLITE_OMIT_JSON #include "sqliteInt.h" From 15bff25111c307c3821f81a1d91c377ee98729e2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 22 Dec 2023 12:57:49 +0000 Subject: [PATCH 68/80] Add a new comment to debugging output routine sqlite3WhereLoopPrint() to remind us of what the various fields of the debug output mean. No changes to code. FossilOrigin-Name: da5f34fd4052432b1ae27bb12e56b358cdc5c1282653d60ed0f0fe62f727e4ee --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 12 ++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index ef0d0902d3..582dc6f8bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sinternal\score-developer-only\sdocumentation\sof\sthe\sJSONB\sformat. -D 2023-12-21T18:08:05.881 +C Add\sa\snew\scomment\sto\sdebugging\soutput\sroutine\ssqlite3WhereLoopPrint()\sto\nremind\sus\sof\swhat\sthe\svarious\sfields\sof\sthe\sdebug\soutput\smean.\s\sNo\schanges\nto\scode. +D 2023-12-22T12:57:49.653 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -822,7 +822,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c e80c22f033edc5bdaf992e32bb740831d90a19e8bc3e68523a27add2478c1f65 +F src/where.c 3c171949ddb5994b1bcf571130bc125fb3197fd1c86ed47c55c9fc0d8d7d0a67 F src/whereInt.h 4b38c5889514e3aead3f27d0ee9a26e47c3f150efc59e2a8b4e3bc8835e4d7a1 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 029a05cd2928d43d81e4549cce5388c432e2c9e75e3fa0b2fe6e91021b2fb9ac -R 37c4deae61c278596f4d5133c7cb9761 +P 4d30478863b2a60512010de9ec6e3099bfaf75d4afee20acec536713fe94334d +R bb6cc7c6adcae554d705e47acb1f531c U drh -Z eab6e0f06a97a969bebac69bfb9fe946 +Z 17fb9a9e25ef316e762bdf861f70c582 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 39fbf7193d..5eccf3624f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d30478863b2a60512010de9ec6e3099bfaf75d4afee20acec536713fe94334d \ No newline at end of file +da5f34fd4052432b1ae27bb12e56b358cdc5c1282653d60ed0f0fe62f727e4ee \ No newline at end of file diff --git a/src/where.c b/src/where.c index 15f206a099..a1545ce0b0 100644 --- a/src/where.c +++ b/src/where.c @@ -2245,6 +2245,18 @@ void sqlite3WhereClausePrint(WhereClause *pWC){ #ifdef WHERETRACE_ENABLED /* ** Print a WhereLoop object for debugging purposes +** +** Format example: +** +** .--- Position in WHERE clause rSetup, rRun, nOut ---. +** | | +** | .--- selfMask nTerm ------. | +** | | | | +** | | .-- prereq Idx wsFlags----. | | +** | | | Name | | | +** | | | __|__ nEq ---. ___|__ | __|__ +** | / \ / \ / \ | / \ / \ / \ +** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31 */ void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ WhereInfo *pWInfo = pWC->pWInfo; From 0810150532cf54f5b6945e9a650468f13298373c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Dec 2023 14:47:30 +0000 Subject: [PATCH 69/80] Fix a usan complaint about signed integer overflow. FossilOrigin-Name: e65907e0279f4814ec957f0790777d8b94a86926cd27c52442b311b27efc0185 --- ext/fts5/fts5_index.c | 4 +- ext/fts5/test/fts5secure3.test | 129 +++++++++++++++++---------------- manifest | 16 ++-- manifest.uuid | 2 +- 4 files changed, 78 insertions(+), 73 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 8a77523f6d..3f88b2af4e 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -1547,9 +1547,9 @@ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ } if( iOffnn ){ - i64 iVal; + u64 iVal; pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; - iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal); + iOff += fts5GetVarint(&pData->p[iOff], &iVal); pLvl->iRowid += iVal; pLvl->iOff = iOff; }else{ diff --git a/ext/fts5/test/fts5secure3.test b/ext/fts5/test/fts5secure3.test index bc56e08200..49347144f3 100644 --- a/ext/fts5/test/fts5secure3.test +++ b/ext/fts5/test/fts5secure3.test @@ -86,71 +86,76 @@ do_execsql_test 2.8 { # Tests with large/small rowid values. # -reset_db - -expr srand(0) - -set vocab { - Popper Poppins Popsicle Porfirio Porrima Porsche - Porter Portia Portland Portsmouth Portugal Portuguese - Poseidon Post PostgreSQL Potemkin Potomac Potsdam - Pottawatomie Potter Potts Pound Poussin Powell - PowerPC PowerPoint Powers Powhatan Poznan Prada - Prado Praetorian Prague Praia Prakrit Pratchett - Pratt Pravda Praxiteles Preakness Precambrian Preminger - Premyslid Prensa Prentice Pres Presbyterian Presbyterianism -} -proc newdoc {} { - for {set i 0} {$i<8} {incr i} { - lappend ret [lindex $::vocab [expr int(abs(rand()) * [llength $::vocab])]] +foreach {tn cfg} { + 1 "" + 2 "INSERT INTO fff(fff, rank) VALUES('secure-delete', 1)" +} { + reset_db + + expr srand(0) + + set vocab { + Popper Poppins Popsicle Porfirio Porrima Porsche + Porter Portia Portland Portsmouth Portugal Portuguese + Poseidon Post PostgreSQL Potemkin Potomac Potsdam + Pottawatomie Potter Potts Pound Poussin Powell + PowerPC PowerPoint Powers Powhatan Poznan Prada + Prado Praetorian Prague Praia Prakrit Pratchett + Pratt Pravda Praxiteles Preakness Precambrian Preminger + Premyslid Prensa Prentice Pres Presbyterian Presbyterianism } - set ret -} -db func newdoc newdoc - -do_execsql_test 3.0 { - CREATE VIRTUAL TABLE fff USING fts5(y); - INSERT INTO fff(fff, rank) VALUES('pgsz', 64); - - WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) - INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; - - WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) - INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; - - WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) - INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; - - INSERT INTO fff(fff, rank) VALUES('secure-delete', 1); -} - -proc lshuffle {in} { - set out [list] - while {[llength $in]>0} { - set idx [expr int(abs(rand()) * [llength $in])] - lappend out [lindex $in $idx] - set in [lreplace $in $idx $idx] - } - set out -} - -#dump fff - -set iTest 1 -foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] { - #if {$iTest==1} { dump fff } - #if {$iTest==1} { breakpoint } - do_execsql_test 3.1.$iTest.$ii { - DELETE FROM fff WHERE rowid=$ii; - } - #if {$iTest==1} { dump fff } - if {($iTest % 20)==0} { - do_execsql_test 3.1.$iTest.$ii.ic { - INSERT INTO fff(fff) VALUES('integrity-check'); + proc newdoc {} { + for {set i 0} {$i<8} {incr i} { + lappend ret [lindex $::vocab [expr int(abs(rand()) * [llength $::vocab])]] } + set ret + } + db func newdoc newdoc + + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE fff USING fts5(y); + INSERT INTO fff(fff, rank) VALUES('pgsz', 64); + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + } + + execsql $cfg + + proc lshuffle {in} { + set out [list] + while {[llength $in]>0} { + set idx [expr int(abs(rand()) * [llength $in])] + lappend out [lindex $in $idx] + set in [lreplace $in $idx $idx] + } + set out + } + + #dump fff + + set iTest 1 + foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] { + #if {$iTest==1} { dump fff } + #if {$iTest==1} { breakpoint } + do_execsql_test 3.$tn.1.$iTest.$ii { + DELETE FROM fff WHERE rowid=$ii; + } + #if {$iTest==1} { dump fff } + if {($iTest % 20)==0} { + do_execsql_test 3.$tn.1.$iTest.$ii.ic { + INSERT INTO fff(fff) VALUES('integrity-check'); + } + } + #if {$iTest==1} { break } + incr iTest } - #if {$iTest==1} { break } - incr iTest } #execsql_pp { SELECT rowid FROM fff('post') ORDER BY rowid ASC } diff --git a/manifest b/manifest index 582dc6f8bd..f89671a85d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\snew\scomment\sto\sdebugging\soutput\sroutine\ssqlite3WhereLoopPrint()\sto\nremind\sus\sof\swhat\sthe\svarious\sfields\sof\sthe\sdebug\soutput\smean.\s\sNo\schanges\nto\scode. -D 2023-12-22T12:57:49.653 +C Fix\sa\susan\scomplaint\sabout\ssigned\sinteger\soverflow. +D 2023-12-22T14:47:30.926 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -97,7 +97,7 @@ F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c 248ecadbacdbeb85c433f907e57bb91224678c829e57ecf098e3543b5df8c3f9 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c ed206045ff0f2226d870fa41fba45f738c0cc953ab74ba68477091b9a574ccd3 +F ext/fts5/fts5_index.c cf00132cf9c81e2be2e98f2c96d7c81de07935177e9868750b620c9bd843bb96 F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933 F ext/fts5/fts5_storage.c f9e31b0d155e9b2c92d5d3a09ad7a56b937fbf1c7f962e10f4ca6281349f3934 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -211,7 +211,7 @@ F ext/fts5/test/fts5rowid.test b8790ec170a8dc1942a15aef3db926a5f3061b1ff17101300 F ext/fts5/test/fts5savepoint.test 050796b24929325cdbbb2fbfe2794816ae95d298e940ae15032200c2f4a73725 F ext/fts5/test/fts5secure.test a02f771742fb2b1b9bdcb4bf523bcf2d0aa1ff597831d40fe3e72aaa6d0ec40f F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc -F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 +F ext/fts5/test/fts5secure3.test 6d066828d225b0dbe5db818d4d6165df7bb70210e68a577e858e8762400d5a23 F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf6be82c88a0ac857fd @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d30478863b2a60512010de9ec6e3099bfaf75d4afee20acec536713fe94334d -R bb6cc7c6adcae554d705e47acb1f531c -U drh -Z 17fb9a9e25ef316e762bdf861f70c582 +P da5f34fd4052432b1ae27bb12e56b358cdc5c1282653d60ed0f0fe62f727e4ee +R 2c6195726c124c3dadd04ddbe5a976af +U dan +Z 5c86608ddefb4b77281d134add392a7d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5eccf3624f..3d8c901bbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da5f34fd4052432b1ae27bb12e56b358cdc5c1282653d60ed0f0fe62f727e4ee \ No newline at end of file +e65907e0279f4814ec957f0790777d8b94a86926cd27c52442b311b27efc0185 \ No newline at end of file From 09e6c82d56a642f2b213c55fbe33b09d23b179b1 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 22 Dec 2023 15:41:13 +0000 Subject: [PATCH 70/80] Update #ifdef checks in pager.c and util.c to account for [0462a2612d1fc1d0] to resolve the build problem reported in [forum:9819032aac|forum post 9819032aac]. FossilOrigin-Name: 0f22d809a1c6c80e381f6bcd931fe4ec36dca0e28d07ab4f4f7f83c813424f60 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 2 +- src/util.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f89671a85d..b268a279b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\susan\scomplaint\sabout\ssigned\sinteger\soverflow. -D 2023-12-22T14:47:30.926 +C Update\s#ifdef\schecks\sin\spager.c\sand\sutil.c\sto\saccount\sfor\s[0462a2612d1fc1d0]\sto\sresolve\sthe\sbuild\sproblem\sreported\sin\s[forum:9819032aac|forum\spost\s9819032aac]. +D 2023-12-22T15:41:13.699 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -724,7 +724,7 @@ F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d87210 F src/os_unix.c 1672c708df279fca1b6ba619cbb26a88baa7913b21dda95817290d76666a9688 F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c dc0cccda12d1675f461ef47090ec6bd7bb7611e77dbf5e0796667120d9b67f37 +F src/pager.c fef16beb072d934ca4f3ef496b2b65703fe2291c4b858782a94cf031c626e816 F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a F src/parse.y 020d80386eb216ec9520549106353c517d2bbc89be28752ffdca649a9eaf56ec F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 @@ -805,7 +805,7 @@ F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e -F src/util.c b22cc9f203a8c0b9ee5338a67f8860347d14845864c10248bebe84518a781677 +F src/util.c 078f040366d5bd5f47658d045f901c768c1c636c6eaea121f3a1cbd63c3edb5b F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 96ac876e57f480bd35ec8d74ed992bca6ae9deebe8b527a3a718e7b4714d6c2e F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P da5f34fd4052432b1ae27bb12e56b358cdc5c1282653d60ed0f0fe62f727e4ee -R 2c6195726c124c3dadd04ddbe5a976af -U dan -Z 5c86608ddefb4b77281d134add392a7d +P e65907e0279f4814ec957f0790777d8b94a86926cd27c52442b311b27efc0185 +R f2929f99f64e2aabe695286ebcbb1f7d +U stephan +Z 9fe589dab0cefe8e50c14fab29339e68 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3d8c901bbc..d8488a85cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e65907e0279f4814ec957f0790777d8b94a86926cd27c52442b311b27efc0185 \ No newline at end of file +0f22d809a1c6c80e381f6bcd931fe4ec36dca0e28d07ab4f4f7f83c813424f60 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 681aa55836..f779d74573 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7792,7 +7792,7 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif -#ifdef SQLITE_USE_SEH +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) int sqlite3PagerWalSystemErrno(Pager *pPager){ return sqlite3WalSystemErrno(pPager->pWal); } diff --git a/src/util.c b/src/util.c index e9c7cccb03..207b901bad 100644 --- a/src/util.c +++ b/src/util.c @@ -141,7 +141,7 @@ void sqlite3ErrorClear(sqlite3 *db){ */ void sqlite3SystemError(sqlite3 *db, int rc){ if( rc==SQLITE_IOERR_NOMEM ) return; -#ifdef SQLITE_USE_SEH +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) if( rc==SQLITE_IOERR_IN_PAGE ){ int ii; int iErr; From ba5043f818e1232f2d7dbfecb77bd7418ca98ee3 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Dec 2023 16:03:45 +0000 Subject: [PATCH 71/80] Add the -fno-sanitize-recover=undefined to the sanitizer builds used for sdevtest and release testing. To ensure that any test that provokes undefined behaviour fails. FossilOrigin-Name: 89563311adb0ab7c7a3eadb11c2e27fbca50c56fce8ca616628facbc00d72b88 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/testrunner_data.tcl | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b268a279b0..6f3470b5d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\s#ifdef\schecks\sin\spager.c\sand\sutil.c\sto\saccount\sfor\s[0462a2612d1fc1d0]\sto\sresolve\sthe\sbuild\sproblem\sreported\sin\s[forum:9819032aac|forum\spost\s9819032aac]. -D 2023-12-22T15:41:13.699 +C Add\sthe\s-fno-sanitize-recover=undefined\sto\sthe\ssanitizer\sbuilds\sused\sfor\ssdevtest\sand\srelease\stesting.\sTo\sensure\sthat\sany\stest\sthat\sprovokes\sundefined\sbehaviour\sfails. +D 2023-12-22T16:03:45.305 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1667,7 +1667,7 @@ F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede F test/testrunner.tcl 8e2a5c7550b78d3283eee6103104ae2bcf56aa1df892dbd1608f27b93ebf4de8 -F test/testrunner_data.tcl 72bbd60e8ffbe5694cf871cbe8f8f6e542c9f1e6a33765309331aeb5e4f16553 +F test/testrunner_data.tcl 7ffd951527bbc614e723fd8d123b6834321878530696adecfdf6035100bac64e F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e65907e0279f4814ec957f0790777d8b94a86926cd27c52442b311b27efc0185 -R f2929f99f64e2aabe695286ebcbb1f7d -U stephan -Z 9fe589dab0cefe8e50c14fab29339e68 +P 0f22d809a1c6c80e381f6bcd931fe4ec36dca0e28d07ab4f4f7f83c813424f60 +R 0b3afe428e241ba40819bcf8993b604e +U dan +Z 0e99aa8a06949f0b1d93d30c00bd8ab5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d8488a85cb..77b11a7374 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f22d809a1c6c80e381f6bcd931fe4ec36dca0e28d07ab4f4f7f83c813424f60 \ No newline at end of file +89563311adb0ab7c7a3eadb11c2e27fbca50c56fce8ca616628facbc00d72b88 \ No newline at end of file diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index 6ca2a80f7a..984c6d8272 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -100,11 +100,11 @@ namespace eval trd { } set build(All-Sanitize) { -DSQLITE_OMIT_LOOKASIDE=1 - --enable-all -fsanitize=address,undefined + --enable-all -fsanitize=address,undefined -fno-sanitize-recover=undefined } set build(Sanitize) { - CC=clang -fsanitize=address,undefined + CC=clang -fsanitize=address,undefined -fno-sanitize-recover=undefined -DSQLITE_ENABLE_STAT4 -DSQLITE_OMIT_LOOKASIDE=1 -DCONFIG_SLOWDOWN_FACTOR=5.0 From 8a630c2aa38092645092a81c36e715e43d64a151 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 22 Dec 2023 21:22:55 +0000 Subject: [PATCH 72/80] Change parameters on a debugging function to include "const". FossilOrigin-Name: 94c3e1110c6590261bd30ba317fba4dd94023d69b81a94f4b216cce748fe7489 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 2 +- src/whereInt.h | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 6f3470b5d4..0d989bc96a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s-fno-sanitize-recover=undefined\sto\sthe\ssanitizer\sbuilds\sused\sfor\ssdevtest\sand\srelease\stesting.\sTo\sensure\sthat\sany\stest\sthat\sprovokes\sundefined\sbehaviour\sfails. -D 2023-12-22T16:03:45.305 +C Change\sparameters\son\sa\sdebugging\sfunction\sto\sinclude\s"const". +D 2023-12-22T21:22:55.369 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -822,8 +822,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 3c171949ddb5994b1bcf571130bc125fb3197fd1c86ed47c55c9fc0d8d7d0a67 -F src/whereInt.h 4b38c5889514e3aead3f27d0ee9a26e47c3f150efc59e2a8b4e3bc8835e4d7a1 +F src/where.c a97ed10fccacadff0377243ec277046dec572b86e8819621d2851de95210e1e9 +F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 F src/window.c 5b1387d59df30d481ed14cceef5f4d1dab1f8752aa106ba72c8b62777bd139d2 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0f22d809a1c6c80e381f6bcd931fe4ec36dca0e28d07ab4f4f7f83c813424f60 -R 0b3afe428e241ba40819bcf8993b604e -U dan -Z 0e99aa8a06949f0b1d93d30c00bd8ab5 +P 89563311adb0ab7c7a3eadb11c2e27fbca50c56fce8ca616628facbc00d72b88 +R b050502a9d7d16c5a5f212017732a147 +U drh +Z 4b93a4c74d01e1f20841ce3c527299a7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 77b11a7374..97aa18321b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89563311adb0ab7c7a3eadb11c2e27fbca50c56fce8ca616628facbc00d72b88 \ No newline at end of file +94c3e1110c6590261bd30ba317fba4dd94023d69b81a94f4b216cce748fe7489 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a1545ce0b0..80314a5b05 100644 --- a/src/where.c +++ b/src/where.c @@ -2258,7 +2258,7 @@ void sqlite3WhereClausePrint(WhereClause *pWC){ ** | / \ / \ / \ | / \ / \ / \ ** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31 */ -void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ +void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ WhereInfo *pWInfo = pWC->pWInfo; int nb = 1+(pWInfo->pTabList->nSrc+3)/4; SrcItem *pItem = pWInfo->pTabList->a + p->iTab; diff --git a/src/whereInt.h b/src/whereInt.h index 343aca9f36..f3cc5776a0 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -502,7 +502,7 @@ Bitmask sqlite3WhereGetMask(WhereMaskSet*,int); #ifdef WHERETRACE_ENABLED void sqlite3WhereClausePrint(WhereClause *pWC); void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm); -void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC); +void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC); #endif WhereTerm *sqlite3WhereFindTerm( WhereClause *pWC, /* The WHERE clause to be searched */ From 4ed0b117f3ddfe97434dbd4ff47b9c782076cd0d Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 23 Dec 2023 11:31:47 +0000 Subject: [PATCH 73/80] Add debugging output routines sqlite3ShowWhereLoop(X) and sqlite3ShowWhereLoopList(X) that can be invoked from a debugger to show a summary of the content of a single WhereLoop object or a list of WhereLoop objects. No change in release builds. FossilOrigin-Name: 5db30bcc338aac1cf081de2deec7e60749ae012e2b6f95ccf745623adb4a31dc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 32 +++++++++++++++++++++++--------- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 0d989bc96a..ee09b1add4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sparameters\son\sa\sdebugging\sfunction\sto\sinclude\s"const". -D 2023-12-22T21:22:55.369 +C Add\sdebugging\soutput\sroutines\ssqlite3ShowWhereLoop(X)\sand\nsqlite3ShowWhereLoopList(X)\sthat\scan\sbe\sinvoked\sfrom\sa\sdebugger\sto\sshow\na\ssummary\sof\sthe\scontent\sof\sa\ssingle\sWhereLoop\sobject\sor\sa\slist\sof\sWhereLoop\nobjects.\s\sNo\schange\sin\srelease\sbuilds. +D 2023-12-23T11:31:47.742 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -822,7 +822,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c a97ed10fccacadff0377243ec277046dec572b86e8819621d2851de95210e1e9 +F src/where.c d2c3185186aefbb624fcd0632eba944d44e19b83e000474df5d601f3d7c508e2 F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 89563311adb0ab7c7a3eadb11c2e27fbca50c56fce8ca616628facbc00d72b88 -R b050502a9d7d16c5a5f212017732a147 +P 94c3e1110c6590261bd30ba317fba4dd94023d69b81a94f4b216cce748fe7489 +R bf03b252f804ee5df8768da2a38496d3 U drh -Z 4b93a4c74d01e1f20841ce3c527299a7 +Z d7bc6f2c748bddf995593faff855247e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 97aa18321b..8e28a9ae73 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -94c3e1110c6590261bd30ba317fba4dd94023d69b81a94f4b216cce748fe7489 \ No newline at end of file +5db30bcc338aac1cf081de2deec7e60749ae012e2b6f95ccf745623adb4a31dc \ No newline at end of file diff --git a/src/where.c b/src/where.c index 80314a5b05..cae3dd0749 100644 --- a/src/where.c +++ b/src/where.c @@ -2259,15 +2259,20 @@ void sqlite3WhereClausePrint(WhereClause *pWC){ ** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31 */ void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ - WhereInfo *pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+3)/4; - SrcItem *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pTab; - Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; - sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, - p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); - sqlite3DebugPrintf(" %12s", - pItem->zAlias ? pItem->zAlias : pTab->zName); + if( pWC ){ + WhereInfo *pWInfo = pWC->pWInfo; + int nb = 1+(pWInfo->pTabList->nSrc+3)/4; + SrcItem *pItem = pWInfo->pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); + sqlite3DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + }else{ + sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d", + p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab); + } if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ const char *zName; if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ @@ -2304,6 +2309,15 @@ void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ } } } +void sqlite3ShowWhereLoop(const WhereLoop *p){ + sqlite3WhereLoopPrint(p, 0); +} +void sqlite3ShowWhereLoopList(const WhereLoop *p){ + while( p ){ + sqlite3WhereLoopPrint(p, 0); + p = p->pNextLoop; + } +} #endif /* From 57c98747cb812a12db4931dcbb38e39fc0b0739f Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 23 Dec 2023 19:03:50 +0000 Subject: [PATCH 74/80] Improvements to the query planner to address the inefficiency described by [forum/forumpost/2568d1f6e6|forum post 2568d1f6e6]. FossilOrigin-Name: 72fcc12cda910a0e3f7875eb3d117b2a5608705c97703985427a02960f1ab5c5 --- manifest | 14 ++++++------ manifest.uuid | 2 +- src/where.c | 58 ++++++++++++++++++++++++++++++------------------ test/where3.test | 22 ++++++++++++++++++ 4 files changed, 66 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index ee09b1add4..7ecb2cb139 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sdebugging\soutput\sroutines\ssqlite3ShowWhereLoop(X)\sand\nsqlite3ShowWhereLoopList(X)\sthat\scan\sbe\sinvoked\sfrom\sa\sdebugger\sto\sshow\na\ssummary\sof\sthe\scontent\sof\sa\ssingle\sWhereLoop\sobject\sor\sa\slist\sof\sWhereLoop\nobjects.\s\sNo\schange\sin\srelease\sbuilds. -D 2023-12-23T11:31:47.742 +C Improvements\sto\sthe\squery\splanner\sto\saddress\sthe\sinefficiency\sdescribed\nby\s[forum/forumpost/2568d1f6e6|forum\spost\s2568d1f6e6]. +D 2023-12-23T19:03:50.123 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -822,7 +822,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c d2c3185186aefbb624fcd0632eba944d44e19b83e000474df5d601f3d7c508e2 +F src/where.c fa15b0763891ed15a8b1beff5577500514af952ed321ba4cf24147fe57728f86 F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 @@ -1970,7 +1970,7 @@ F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2a F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 F test/where.test 59abb854eee24f166b5f7ba9d17eb250abc59ce0a66c48912ffb10763648196d F test/where2.test 03c21a11e7b90e2845fc3c8b4002fc44cc2797fa74c86ee47d70bd7ea4f29ed6 -F test/where3.test 5b4ffc0ac2ea0fe92f02b1244b7531522fe4d7bccf6fa8741d54e82c10e67753 +F test/where3.test 4ccb156ae33de86414a52775a6f590a9d60ba2cbc7a93a24fa331b7bcf5b6030 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 94c3e1110c6590261bd30ba317fba4dd94023d69b81a94f4b216cce748fe7489 -R bf03b252f804ee5df8768da2a38496d3 +P 5db30bcc338aac1cf081de2deec7e60749ae012e2b6f95ccf745623adb4a31dc +R 448157c6fb649e47b8145db2a0559aa3 U drh -Z d7bc6f2c748bddf995593faff855247e +Z 1a62d7dd3eca74dd62a870ea245ac4e3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8e28a9ae73..99f3b056e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5db30bcc338aac1cf081de2deec7e60749ae012e2b6f95ccf745623adb4a31dc \ No newline at end of file +72fcc12cda910a0e3f7875eb3d117b2a5608705c97703985427a02960f1ab5c5 \ No newline at end of file diff --git a/src/where.c b/src/where.c index cae3dd0749..0b56de6c16 100644 --- a/src/where.c +++ b/src/where.c @@ -2430,46 +2430,60 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Return TRUE if all of the following are true: +** Return TRUE if X is a proper subset of Y but is of equal or less cost. +** In other words, return true if all constraints of X are also part of Y +** and Y has additional constraints that might speed the search that X lacks +** but the cost of running X is not more than the cost of running Y. ** -** (1) X has the same or lower cost, or returns the same or fewer rows, -** than Y. -** (2) X uses fewer WHERE clause terms than Y -** (3) Every WHERE clause term used by X is also used by Y -** (4) X skips at least as many columns as Y -** (5) If X is a covering index, than Y is too +** In other words, return true if the cost relationwship between X and Y +** is inverted and needs to be adjusted. ** -** Conditions (2) and (3) mean that X is a "proper subset" of Y. -** If X is a proper subset of Y then Y is a better choice and ought -** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. Constraint (4) -** was added because if X uses skip-scan less than Y it still might -** deserve a lower cost even if it is a proper subset of Y. Constraint (5) -** was added because a covering index probably deserves to have a lower cost -** than a non-covering index even if it is a proper subset. +** Case 1: +** +** (1a) X and Y use the same index. +** (1b) X has fewer == terms than Y +** (1c) Neither X nor Y use skip-scan +** (1d) X does not have a a greater cost than Y +** +** Case 2: +** +** (2a) X has the same or lower cost, or returns the same or fewer rows, +** than Y. +** (2b) X uses fewer WHERE clause terms than Y +** (2c) Every WHERE clause term used by X is also used by Y +** (2d) X skips at least as many columns as Y +** (2e) If X is a covering index, than Y is too */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; - if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ - return 0; /* X is not a subset of Y */ + if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; /* (1d) and (2a) */ + assert( (pX->wsFlags & WHERE_VIRTUALTABLE)==0 ); + assert( (pY->wsFlags & WHERE_VIRTUALTABLE)==0 ); + if( pX->u.btree.nEq < pY->u.btree.nEq /* (1b) */ + && pX->u.btree.pIndex==pY->u.btree.pIndex /* (1a) */ + && pX->nSkip==0 && pY->nSkip==0 /* (1c) */ + ){ + return 1; /* Case 1 is true */ } - if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; - if( pY->nSkip > pX->nSkip ) return 0; + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* (2b) */ + } + if( pY->nSkip > pX->nSkip ) return 0; /* (2d) */ for(i=pX->nLTerm-1; i>=0; i--){ if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } - if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + if( j<0 ) return 0; /* (2c) */ } if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){ - return 0; /* Constraint (5) */ + return 0; /* (2e) */ } - return 1; /* All conditions meet */ + return 1; /* Case 2 is true */ } /* diff --git a/test/where3.test b/test/where3.test index 20af995cfe..f574c0d55f 100644 --- a/test/where3.test +++ b/test/where3.test @@ -492,5 +492,27 @@ foreach disabled_opt {none omit-noop-join all} { } {123} } +# 2023-12-23 +# https://sqlite.org/forum/forumpost/2568d1f6e6 +# +# Index usage should be "x=? and y=?" - equality on both values. +# Not: "x=? AND y>?" - inequality on "y" +# +reset_db +do_execsql_test where3-8.1 { + CREATE TABLE t1(a,b,c,d); INSERT INTO t1 VALUES(1,2,3,4); + CREATE TABLE t2(x,y); INSERT INTO t2 VALUES(3,4); + CREATE INDEX t2xy ON t2(x,y); + SELECT 1 FROM t1 JOIN t2 ON x=c AND y=d WHERE d>0; +} 1 +do_eqp_test where3-8.2 { + SELECT 1 FROM t1 JOIN t2 ON x=c AND y=d WHERE d>0; +} { + QUERY PLAN + |--SCAN t1 + `--SEARCH t2 USING COVERING INDEX t2xy (x=? AND y=?) +} + + finish_test From c1805ab222214ddce9779691543117868291c7a0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 24 Dec 2023 11:31:20 +0000 Subject: [PATCH 75/80] Avoid signed integer overflow during integrity_check of FTS5. FossilOrigin-Name: 5937df3b25799eceaadfb04d7226c9995d44c8d8edb5ac3ad02af9d7e3570726 --- ext/fts5/fts5_index.c | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 3f88b2af4e..8eb8f328f6 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -7944,7 +7944,7 @@ static void fts5IndexIntegrityCheckEmpty( } static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ - int iTermOff = 0; + i64 iTermOff = 0; int ii; Fts5Buffer buf1 = {0,0,0}; @@ -7953,7 +7953,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ ii = pLeaf->szLeaf; while( iinn && p->rc==SQLITE_OK ){ int res; - int iOff; + i64 iOff; int nIncr; ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); diff --git a/manifest b/manifest index 7ecb2cb139..fef344e241 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\squery\splanner\sto\saddress\sthe\sinefficiency\sdescribed\nby\s[forum/forumpost/2568d1f6e6|forum\spost\s2568d1f6e6]. -D 2023-12-23T19:03:50.123 +C Avoid\ssigned\sinteger\soverflow\sduring\sintegrity_check\sof\sFTS5. +D 2023-12-24T11:31:20.545 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -97,7 +97,7 @@ F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c 248ecadbacdbeb85c433f907e57bb91224678c829e57ecf098e3543b5df8c3f9 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c cf00132cf9c81e2be2e98f2c96d7c81de07935177e9868750b620c9bd843bb96 +F ext/fts5/fts5_index.c bb1965c3965f6fe5f64160bf1c0694a9684a790a783f293a76da1d38d319b258 F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933 F ext/fts5/fts5_storage.c f9e31b0d155e9b2c92d5d3a09ad7a56b937fbf1c7f962e10f4ca6281349f3934 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5db30bcc338aac1cf081de2deec7e60749ae012e2b6f95ccf745623adb4a31dc -R 448157c6fb649e47b8145db2a0559aa3 +P 72fcc12cda910a0e3f7875eb3d117b2a5608705c97703985427a02960f1ab5c5 +R 9dd3dbd9eae73b546b307507a5926f2d U drh -Z 1a62d7dd3eca74dd62a870ea245ac4e3 +Z 09eda7ff6cea8413533fe43e801321e8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 99f3b056e3..a7bc0ee567 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72fcc12cda910a0e3f7875eb3d117b2a5608705c97703985427a02960f1ab5c5 \ No newline at end of file +5937df3b25799eceaadfb04d7226c9995d44c8d8edb5ac3ad02af9d7e3570726 \ No newline at end of file From 5a85cf5879d1c875bd93ef3a572a59db5b57a96c Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 24 Dec 2023 11:43:49 +0000 Subject: [PATCH 76/80] Fix harmless compiler warnings associated with [5db30bcc338aac1c] FossilOrigin-Name: e55d1c2333f35fc20615aa83a7843d08cae7945710a2156d44eee0cc37d90ade --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 9 +++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index fef344e241..a8e0e230af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssigned\sinteger\soverflow\sduring\sintegrity_check\sof\sFTS5. -D 2023-12-24T11:31:20.545 +C Fix\sharmless\scompiler\swarnings\sassociated\swith\s[5db30bcc338aac1c] +D 2023-12-24T11:43:49.460 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -822,7 +822,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c fa15b0763891ed15a8b1beff5577500514af952ed321ba4cf24147fe57728f86 +F src/where.c 0bfab37c7f787e320a8010e51ae97c2e51964d3b3a24fbc246b8e8fee50de4e9 F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 72fcc12cda910a0e3f7875eb3d117b2a5608705c97703985427a02960f1ab5c5 -R 9dd3dbd9eae73b546b307507a5926f2d +P 5937df3b25799eceaadfb04d7226c9995d44c8d8edb5ac3ad02af9d7e3570726 +R f383e5dfc66b947bc00ea28ad85dc8b1 U drh -Z 09eda7ff6cea8413533fe43e801321e8 +Z 93113c6d4b8e5085563bd90cb3fc82d6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a7bc0ee567..4e62bcd286 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5937df3b25799eceaadfb04d7226c9995d44c8d8edb5ac3ad02af9d7e3570726 \ No newline at end of file +e55d1c2333f35fc20615aa83a7843d08cae7945710a2156d44eee0cc37d90ade \ No newline at end of file diff --git a/src/where.c b/src/where.c index 0b56de6c16..bffca3413d 100644 --- a/src/where.c +++ b/src/where.c @@ -2310,11 +2310,11 @@ void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ } } void sqlite3ShowWhereLoop(const WhereLoop *p){ - sqlite3WhereLoopPrint(p, 0); + if( p ) sqlite3WhereLoopPrint(p, 0); } void sqlite3ShowWhereLoopList(const WhereLoop *p){ while( p ){ - sqlite3WhereLoopPrint(p, 0); + sqlite3ShowWhereLoop(p); p = p->pNextLoop; } } @@ -6643,6 +6643,11 @@ whereBeginError: pParse->nQueryLoop = pWInfo->savedNQueryLoop; whereInfoFree(db, pWInfo); } +#ifdef WHERETRACE_ENABLED + /* Prevent harmless compiler warnings about debugging routines + ** being declared but never used */ + sqlite3ShowWhereLoopList(0); +#endif /* WHERETRACE_ENABLED */ return 0; } From 71a32aede3a34b65bf8a23682b28632bb54cf983 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 24 Dec 2023 12:02:36 +0000 Subject: [PATCH 77/80] Remove an ALWAYS() added in [c50e6c2ace49d092] because it is sometimes false. dbsqlfuzz c393a4f783d42efd9552772110aff7e5d937f15e. FossilOrigin-Name: b9daf37e57cde12c4de271a2b1995e8e91b6411f8c2e8882e536241929609b3a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a8e0e230af..c02d741663 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sassociated\swith\s[5db30bcc338aac1c] -D 2023-12-24T11:43:49.460 +C Remove\san\sALWAYS()\sadded\sin\s[c50e6c2ace49d092]\sbecause\sit\sis\ssometimes\sfalse.\ndbsqlfuzz\sc393a4f783d42efd9552772110aff7e5d937f15e. +D 2023-12-24T12:02:36.449 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -687,7 +687,7 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c 125e6966297727b5d647d73893873b5d0730377f73208dc70e39dea9aa29f366 +F src/expr.c 3381ee4c9aa7ccde22a2a7f35ce343925a7a25d96bdc943649131f9decdebad2 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5937df3b25799eceaadfb04d7226c9995d44c8d8edb5ac3ad02af9d7e3570726 -R f383e5dfc66b947bc00ea28ad85dc8b1 +P e55d1c2333f35fc20615aa83a7843d08cae7945710a2156d44eee0cc37d90ade +R 3402eea35cb157b90bc34e52efc1a052 U drh -Z 93113c6d4b8e5085563bd90cb3fc82d6 +Z 3f09f0e552099f2a662787477373f861 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4e62bcd286..6f2035ba05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e55d1c2333f35fc20615aa83a7843d08cae7945710a2156d44eee0cc37d90ade \ No newline at end of file +b9daf37e57cde12c4de271a2b1995e8e91b6411f8c2e8882e536241929609b3a \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 11d052feea..f9b280bbc5 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6476,7 +6476,7 @@ static int exprRefToSrcList(Walker *pWalker, Expr *pExpr){ int i; struct RefSrcList *p = pWalker->u.pRefSrcList; SrcList *pSrc = p->pRef; - int nSrc = ALWAYS(pSrc) ? pSrc->nSrc : 0; + int nSrc = pSrc ? pSrc->nSrc : 0; for(i=0; iiTable==pSrc->a[i].iCursor ){ pWalker->eCode |= 1; From b2b74908621dacdba7077d38830c59f33ca77128 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 26 Dec 2023 13:20:57 +0000 Subject: [PATCH 78/80] Improved handling of malformed unicode within JSON strings. FossilOrigin-Name: e252bdf5f5de26ba8e2bcc6b0ad94121ed6fc4d86c02fe4a2a058ada93747beb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 26 ++++++++++++++++++-------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c02d741663..99482bbc2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sALWAYS()\sadded\sin\s[c50e6c2ace49d092]\sbecause\sit\sis\ssometimes\sfalse.\ndbsqlfuzz\sc393a4f783d42efd9552772110aff7e5d937f15e. -D 2023-12-24T12:02:36.449 +C Improved\shandling\sof\smalformed\sunicode\swithin\sJSON\sstrings. +D 2023-12-26T13:20:57.593 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -697,7 +697,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 498cbe3346f216e3119e6d01585c3bc09994612c16459f2fa682a1cec61b1cf7 +F src/json.c bc90605da937ca0cd72ff0492216fbb38fd8f9025e6344499f9db235be98e36f F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e55d1c2333f35fc20615aa83a7843d08cae7945710a2156d44eee0cc37d90ade -R 3402eea35cb157b90bc34e52efc1a052 +P b9daf37e57cde12c4de271a2b1995e8e91b6411f8c2e8882e536241929609b3a +R 8e5515c216d097520f23a07a88eb7f34 U drh -Z 3f09f0e552099f2a662787477373f861 +Z acb4ee2fdca29b1d699d6a53a9b5165e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6f2035ba05..aa146ba31b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9daf37e57cde12c4de271a2b1995e8e91b6411f8c2e8882e536241929609b3a \ No newline at end of file +e252bdf5f5de26ba8e2bcc6b0ad94121ed6fc4d86c02fe4a2a058ada93747beb \ No newline at end of file diff --git a/src/json.c b/src/json.c index e4f658ca7f..42d6155fb6 100644 --- a/src/json.c +++ b/src/json.c @@ -214,6 +214,12 @@ typedef struct JsonParse JsonParse; #define JSON_CACHE_ID (-429938) /* Cache entry */ #define JSON_CACHE_SIZE 4 /* Max number of cache entries */ +/* +** jsonUnescapeOneChar() returns this invalid code point if it encounters +** a syntax error. +*/ +#define JSON_INVALID_CHAR 0x99999 + /* A cache mapping JSON text into JSONB blobs. ** ** Each cache entry is a JsonParse object with the following restrictions: @@ -1380,7 +1386,7 @@ static u32 jsonbValidityCheck( }else{ u32 c = 0; u32 szC = jsonUnescapeOneChar((const char*)&z[j], k-j, &c); - if( c==0xfffd ) return j+1; + if( c==JSON_INVALID_CHAR ) return j+1; j += szC - 1; } } @@ -2390,19 +2396,23 @@ static u32 jsonBytesToBypass(const char *z, u32 n){ ** Input z[0..n] defines JSON escape sequence including the leading '\\'. ** Decode that escape sequence into a single character. Write that ** character into *piOut. Return the number of bytes in the escape sequence. +** +** If there is a syntax error of some kind (for example too few characters +** after the '\\' to complete the encoding) then *piOut is set to +** JSON_INVALID_CHAR. */ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ assert( n>0 ); assert( z[0]=='\\' ); if( n<2 ){ - *piOut = 0xFFFD; + *piOut = JSON_INVALID_CHAR; return n; } switch( (u8)z[1] ){ case 'u': { u32 v, vlo; if( n<6 ){ - *piOut = 0xFFFD; + *piOut = JSON_INVALID_CHAR; return n; } v = jsonHexToInt4(&z[2]); @@ -2432,7 +2442,7 @@ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ case '\\':{ *piOut = z[1]; return 2; } case 'x': { if( n<4 ){ - *piOut = 0xFFFD; + *piOut = JSON_INVALID_CHAR; return n; } *piOut = (jsonHexToInt(z[2])<<4) | jsonHexToInt(z[3]); @@ -2443,7 +2453,7 @@ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ case '\n': { u32 nSkip = jsonBytesToBypass(z, n); if( nSkip==0 ){ - *piOut = 0xFFFD; + *piOut = JSON_INVALID_CHAR; return n; }else if( nSkip==n ){ *piOut = 0; @@ -2456,7 +2466,7 @@ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ } } default: { - *piOut = 0xFFFD; + *piOut = JSON_INVALID_CHAR; return 2; } } @@ -2930,8 +2940,6 @@ static void jsonReturnFromBlob( u32 szEscape = jsonUnescapeOneChar(&z[iIn], sz-iIn, &v); if( v<=0x7f ){ zOut[iOut++] = (char)v; - }else if( v==0xfffd ){ - /* Silently ignore illegal unicode */ }else if( v<=0x7ff ){ assert( szEscape>=2 ); zOut[iOut++] = (char)(0xc0 | (v>>6)); @@ -2941,6 +2949,8 @@ static void jsonReturnFromBlob( zOut[iOut++] = 0xe0 | (v>>12); zOut[iOut++] = 0x80 | ((v>>6)&0x3f); zOut[iOut++] = 0x80 | (v&0x3f); + }else if( v==JSON_INVALID_CHAR ){ + /* Silently ignore illegal unicode */ }else{ assert( szEscape>=4 ); zOut[iOut++] = 0xf0 | (v>>18); From d82320ac9a60612b8d71f9f72813247c34f69e36 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Dec 2023 15:52:40 +0000 Subject: [PATCH 79/80] Ensure that the xColumnText(), xQueryPhrase() and xPhraseFirstColumn() APIs all return SQLITE_RANGE if they are passed a bad column or phrase number. FossilOrigin-Name: 1a8a9b1c89519d265869251e8b6d3c5db733f0d3a7dea6c7962811a8f1157dff --- ext/fts5/fts5.h | 44 +++++++++++++++++++++++---------- ext/fts5/fts5_aux.c | 6 +++-- ext/fts5/fts5_expr.c | 44 +++++++++++++++++++-------------- ext/fts5/fts5_main.c | 28 ++++++++++++++------- ext/fts5/test/fts5_common.tcl | 20 +++++++++++++++ ext/fts5/test/fts5aux.test | 43 ++++++++++++++++++++++++++++++++ ext/fts5/test/fts5synonym2.test | 2 +- manifest | 26 +++++++++---------- manifest.uuid | 2 +- 9 files changed, 157 insertions(+), 58 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index c7a5e6b7de..250d2ee7e3 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -88,8 +88,11 @@ struct Fts5PhraseIter { ** created with the "columnsize=0" option. ** ** xColumnText: -** This function attempts to retrieve the text of column iCol of the -** current document. If successful, (*pz) is set to point to a buffer +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the text of column iCol of +** the current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values @@ -99,8 +102,10 @@ struct Fts5PhraseIter { ** Returns the number of phrases in the current query expression. ** ** xPhraseSize: -** Returns the number of tokens in phrase iPhrase of the query. Phrases -** are numbered starting from zero. +** If parameter iCol is less than zero, or greater than or equal to the +** number of phrases in the current query, as returned by xPhraseCount, +** 0 is returned. Otherwise, this function returns the number of tokens in +** phrase iPhrase of the query. Phrases are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within @@ -116,12 +121,13 @@ struct Fts5PhraseIter { ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value -** output by xInstCount(). +** output by xInstCount(). If iIdx is less than zero or greater than +** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned. ** -** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. Returns SQLITE_OK if successful, or an error -** code (i.e. SQLITE_NOMEM) if an error occurs. +** first token of the phrase. SQLITE_OK is returned if successful, or an +** error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. @@ -147,6 +153,10 @@ struct Fts5PhraseIter { ** Invoking Api.xUserData() returns a copy of the pointer passed as ** the third argument to pUserData. ** +** If parameter iPhrase is less than zero, or greater than or equal to +** the number of phrases in the query, as returned by xPhraseCount(), +** this function returns SQLITE_RANGE. +** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. @@ -268,17 +278,25 @@ struct Fts5PhraseIter { ** to a buffer containing the requested token, and *pnToken to the ** size of this buffer in bytes. ** +** If iPhrase or iToken are less than zero, or if iPhrase is greater than +** or equal to the number of phrases in the query as reported by +** xPhraseCount(), or if iToken is equal to or greater than the number of +** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken + are both zeroed. +** ** The output text is not a copy of the query text that specified the ** token. It is the output of the tokenizer module. For tokendata=1 ** tables, this includes any embedded 0x00 and trailing data. ** ** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) ** This is used to access token iToken of phrase hit iIdx within the -** current row. Output variable (*ppToken) is set to point to a buffer -** containing the matching document token, and (*pnToken) to the size -** of that buffer in bytes. This API is not available if the specified -** token matches a prefix query term. In that case both output variables -** are always set to 0. +** current row. If iIdx is less than zero or greater than or equal to the +** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, +** output variable (*ppToken) is set to point to a buffer containing the +** matching document token, and (*pnToken) to the size of that buffer in +** bytes. This API is not available if the specified token matches a +** prefix query term. In that case both output variables are always set +** to 0. ** ** The output text is not a copy of the document text that was tokenized. ** It is the output of the tokenizer module. For tokendata=1 tables, this diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index b6919fc9cc..30101fbe25 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -252,8 +252,10 @@ static void fts5HighlightFunction( ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); ctx.iRangeEnd = -1; rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); - - if( ctx.zIn ){ + if( rc==SQLITE_RANGE ){ + sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); + rc = SQLITE_OK; + }else if( ctx.zIn ){ if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); } diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 2f2bf28260..3a7e1d5e82 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1890,8 +1890,12 @@ int sqlite3Fts5ExprClonePhrase( Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */ - pOrig = pExpr->apExprPhrase[iPhrase]; - pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + rc = SQLITE_RANGE; + }else{ + pOrig = pExpr->apExprPhrase[iPhrase]; + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + } if( rc==SQLITE_OK ){ pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase*)); @@ -1918,25 +1922,27 @@ int sqlite3Fts5ExprClonePhrase( } } - if( pOrig->nTerm ){ - int i; /* Used to iterate through phrase terms */ - sCtx.pConfig = pExpr->pConfig; - for(i=0; rc==SQLITE_OK && inTerm; i++){ - int tflags = 0; - Fts5ExprTerm *p; - for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ - rc = fts5ParseTokenize((void*)&sCtx, tflags, p->pTerm,p->nFullTerm,0,0); - tflags = FTS5_TOKEN_COLOCATED; - } - if( rc==SQLITE_OK ){ - sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; - sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + if( rc==SQLITE_OK ){ + if( pOrig->nTerm ){ + int i; /* Used to iterate through phrase terms */ + sCtx.pConfig = pExpr->pConfig; + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + } } + }else{ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); } - }else{ - /* This happens when parsing a token or quoted phrase that contains - ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); } if( rc==SQLITE_OK && ALWAYS(sCtx.pPhrase) ){ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 6d3d147f21..05946d8b64 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1911,7 +1911,10 @@ static int fts5ApiColumnText( ){ int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + if( iCol<0 || iCol>=pTab->pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) || pCsr->ePlan==FTS5_PLAN_SPECIAL ){ *pz = 0; @@ -1936,8 +1939,9 @@ static int fts5CsrPoslist( int rc = SQLITE_OK; int bLive = (pCsr->pSorter==0); - if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ - + if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){ + rc = SQLITE_RANGE; + }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; @@ -1961,15 +1965,21 @@ static int fts5CsrPoslist( CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); } - if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ - Fts5Sorter *pSorter = pCsr->pSorter; - int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); - *pn = pSorter->aIdx[iPhrase] - i1; - *pa = &pSorter->aPoslist[i1]; + if( rc==SQLITE_OK ){ + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + *pn = pSorter->aIdx[iPhrase] - i1; + *pa = &pSorter->aPoslist[i1]; + }else{ + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + } }else{ - *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + *pa = 0; + *pn = 0; } + return rc; } diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 001cad1de2..fda388a6fb 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -61,6 +61,12 @@ proc fts5_test_collist {cmd} { set res } +proc fts5_collist {cmd iPhrase} { + set res [list] + $cmd xPhraseColumnForeach $iPhrase c { lappend res $c } + set res +} + proc fts5_test_columnsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -69,6 +75,10 @@ proc fts5_test_columnsize {cmd} { set res } +proc fts5_columntext {cmd iCol} { + $cmd xColumnText $iCol +} + proc fts5_test_columntext {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -125,6 +135,13 @@ proc fts5_test_queryphrase {cmd} { set res } +proc fts5_queryphrase {cmd iPhrase} { + set cnt [list] + for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 } + $cmd xQueryPhrase $iPhrase [list test_queryphrase_cb cnt] + set cnt +} + proc fts5_test_phrasecount {cmd} { $cmd xPhraseCount } @@ -154,6 +171,9 @@ proc fts5_aux_test_functions {db} { fts5_test_queryphrase fts5_test_phrasecount + fts5_columntext + fts5_queryphrase + fts5_collist } { sqlite3_fts5_create_function $db $f $f } diff --git a/ext/fts5/test/fts5aux.test b/ext/fts5/test/fts5aux.test index b5a13aea1a..5569f48cf3 100644 --- a/ext/fts5/test/fts5aux.test +++ b/ext/fts5/test/fts5aux.test @@ -334,4 +334,47 @@ do_execsql_test 11.2 { SELECT fts5_hitcount(x1) FROM x1('one') LIMIT 1; } {5} +#------------------------------------------------------------------------- +# Test that xColumnText returns SQLITE_RANGE when it should. +# +reset_db +fts5_aux_test_functions db +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + INSERT INTO t1 VALUES('one', 'two', 'three'); + INSERT INTO t1 VALUES('one', 'one', 'one'); + INSERT INTO t1 VALUES('two', 'two', 'two'); + INSERT INTO t1 VALUES('three', 'three', 'three'); +} + +do_catchsql_test 12.1.1 { + SELECT fts5_columntext(t1, -1) FROM t1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.1.2 { + SELECT fts5_columntext(t1, 3) FROM t1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.1.2 { + SELECT fts5_columntext(t1, 1) FROM t1('one AND two'); +} {0 two} + +do_catchsql_test 12.2.1 { + SELECT fts5_queryphrase(t1, -1) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.2.2 { + SELECT fts5_queryphrase(t1, 2) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.2.3 { + SELECT fts5_queryphrase(t1, 1) FROM t1('one AND two'); +} {0 {{1 2 1}}} + +do_catchsql_test 12.3.1 { + SELECT fts5_collist(t1, -1) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.3.2 { + SELECT fts5_collist(t1, 2) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.3.3 { + SELECT fts5_collist(t1, 1) FROM t1('one AND two'); +} {0 1} + finish_test diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test index 2c2705e34f..9fdf757693 100644 --- a/ext/fts5/test/fts5synonym2.test +++ b/ext/fts5/test/fts5synonym2.test @@ -42,7 +42,7 @@ proc fts5_test_bothlist {cmd} { } sqlite3_fts5_create_function db fts5_test_bothlist fts5_test_bothlist -proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] } +proc fts5_rowid {cmd} { expr [$cmd xRowid] } sqlite3_fts5_create_function db fts5_rowid fts5_rowid do_execsql_test 1.$tok.0.1 " diff --git a/manifest b/manifest index 99482bbc2b..6a5b7e182d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\shandling\sof\smalformed\sunicode\swithin\sJSON\sstrings. -D 2023-12-26T13:20:57.593 +C Ensure\sthat\sthe\sxColumnText(),\sxQueryPhrase()\sand\sxPhraseFirstColumn()\sAPIs\sall\sreturn\sSQLITE_RANGE\sif\sthey\sare\spassed\sa\sbad\scolumn\sor\sphrase\snumber. +D 2023-12-26T15:52:40.355 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -90,15 +90,15 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h 9d7867cc63631296462c90bafe57c4881b56e480fa510ffaee6a77049258c714 +F ext/fts5/fts5.h ecba24fed7b359b3a53016bb07e411b3b4c9cdf163aa141006536423a63b611e F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834 -F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad +F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c 248ecadbacdbeb85c433f907e57bb91224678c829e57ecf098e3543b5df8c3f9 +F ext/fts5/fts5_expr.c e5fb9dd9e31e9e6ae9604bdb0d183ecec720964f3b974fc37c43ce73d8833d6d F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c bb1965c3965f6fe5f64160bf1c0694a9684a790a783f293a76da1d38d319b258 -F ext/fts5/fts5_main.c b908696c52410e8383019ac0657c8a5cd0c8f60e78edc169e9b3c4b93f24c933 +F ext/fts5/fts5_main.c 93e5bc8676dbaaec365576d26cdafc5c21f0052f8f68b86ee4de22237c2e2143 F ext/fts5/fts5_storage.c f9e31b0d155e9b2c92d5d3a09ad7a56b937fbf1c7f962e10f4ca6281349f3934 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -109,7 +109,7 @@ F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d0988 F ext/fts5/fts5_vocab.c 209e0c151e108d5f3621fa24b91e9b02f3750ee6c3f9ccec312df39481b68a09 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 8b1848ac2baad10e444e4183034a52050b52d20b3796d9d30e78f01ab0d05583 +F ext/fts5/test/fts5_common.tcl 3378732aae2a7d9a4b9b5c40bde678d4259ca16bd490883325aecc4747bcb384 F ext/fts5/test/fts5aa.test 4db81519863244a3cab35795fe65ab6b592e7970c7409eba098b23ebbfc08d95 F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d @@ -124,7 +124,7 @@ F ext/fts5/test/fts5ak.test f459a64c9d38698af72a7c657ab6349bca96150241dd69fcce75 F ext/fts5/test/fts5al.test 00c4c1c6a1366b73aa48ce2068c634520867c3cf7f5d1676ebbb775ee1f35734 F ext/fts5/test/fts5alter.test 5565f7e4605512b69171ac18ca84398603f9f6456dbe377beeca97e83cc242cd F ext/fts5/test/fts5auto.test 78989e6527ce69c9eddbef7392fea5c10b0010cd2b2ae68eec7bc869c471e691 -F ext/fts5/test/fts5aux.test 3f194345fcd581f49f7fbb2e5495400efcc7d2835b77816328d8283c942f41b8 +F ext/fts5/test/fts5aux.test ed3596469f85a6cff5f6060e0cd9e3f9602051d8db2b497f5d12c85d39f20a62 F ext/fts5/test/fts5auxdata.test eacc97ff04892f1a5f3d4df5a73f8bcbc3955ea1d12c9f24137eb1fc079e7611 F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd F ext/fts5/test/fts5bigpl.test 6466c89b38439f0aba26ac09e232a6b963f29b1cbe1304f6a664fe1e7a8f5fd3 @@ -222,7 +222,7 @@ F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f40 F ext/fts5/test/fts5simple2.test 8dd2389ee75e21a1429fe87e5f8c7d9a97ad1470304a8a2d3ba4b8c3c345fecd F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f F ext/fts5/test/fts5synonym.test 1651815b8008de170e8e600dcacc17521d765482ea8f074ae82cfa870d8bb7fb -F ext/fts5/test/fts5synonym2.test 8f891fc49cc1e8daed727051e77e1f42849c784a6a54bef82564761b2cb3e016 +F ext/fts5/test/fts5synonym2.test e2f6ff68c4fbe12a866a3a87510f553d9dac99bcb74c10b56487c4c0a562fcf5 F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0af27884ffe9cef F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d2153771a8e8a20c721305a43 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b9daf37e57cde12c4de271a2b1995e8e91b6411f8c2e8882e536241929609b3a -R 8e5515c216d097520f23a07a88eb7f34 -U drh -Z acb4ee2fdca29b1d699d6a53a9b5165e +P e252bdf5f5de26ba8e2bcc6b0ad94121ed6fc4d86c02fe4a2a058ada93747beb +R a8a4e2c094005bc85798feb38e88e2ad +U dan +Z 04de1187c12166f6095f68d3113b177a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aa146ba31b..cf1501217e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e252bdf5f5de26ba8e2bcc6b0ad94121ed6fc4d86c02fe4a2a058ada93747beb \ No newline at end of file +1a8a9b1c89519d265869251e8b6d3c5db733f0d3a7dea6c7962811a8f1157dff \ No newline at end of file From d9ac37fc4413dfac97d7bbc261c1984e0ab477b4 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 27 Dec 2023 16:24:53 +0000 Subject: [PATCH 80/80] Fix a problem in the shell tool (not library) causing an out-of-bounds write if an ".open" command failed, then the user pressed ctrl-c to interrupt a query running on the substitute in-memory database. FossilOrigin-Name: 026618b9e321576f616a32e41329066ba629814170c6cfeef35430343f5003f3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6a5b7e182d..692bc45510 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\sxColumnText(),\sxQueryPhrase()\sand\sxPhraseFirstColumn()\sAPIs\sall\sreturn\sSQLITE_RANGE\sif\sthey\sare\spassed\sa\sbad\scolumn\sor\sphrase\snumber. -D 2023-12-26T15:52:40.355 +C Fix\sa\sproblem\sin\sthe\sshell\stool\s(not\slibrary)\scausing\san\sout-of-bounds\swrite\sif\san\s".open"\scommand\sfailed,\sthen\sthe\suser\spressed\sctrl-c\sto\sinterrupt\sa\squery\srunning\son\sthe\ssubstitute\sin-memory\sdatabase. +D 2023-12-27T16:24:53.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -738,7 +738,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c e25f51a473a5f30a0d978e4df2aaa98aeec84eac29ecae1ad4708a6c3e669345 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c f1a81ff4f8e9e76c224e2ab3a4baa799add0db22158c7fcede65d8cc4a6fa2da -F src/shell.c.in 0df801a0445af3fc55c7330bcd8396ca5433154bb0f728edc8be2d6f27764361 +F src/shell.c.in 85f8d52fa4f7773823736dd39d0a268fd739207fcae95883c9ec8ce4af59f7df F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e252bdf5f5de26ba8e2bcc6b0ad94121ed6fc4d86c02fe4a2a058ada93747beb -R a8a4e2c094005bc85798feb38e88e2ad +P 1a8a9b1c89519d265869251e8b6d3c5db733f0d3a7dea6c7962811a8f1157dff +R 9be7ded4b93fb872773060703710555e U dan -Z 04de1187c12166f6095f68d3113b177a +Z 20de7b64b135a6b7a6b0796593663512 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cf1501217e..f5d9fa1a12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a8a9b1c89519d265869251e8b6d3c5db733f0d3a7dea6c7962811a8f1157dff \ No newline at end of file +026618b9e321576f616a32e41329066ba629814170c6cfeef35430343f5003f3 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 26b44e05a9..8f83890525 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5319,7 +5319,6 @@ static void open_db(ShellState *p, int openFlags){ break; } } - globalDb = p->db; if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ eputf("Error: unable to open database \"%s\": %s\n", zDbFilename, sqlite3_errmsg(p->db)); @@ -5336,6 +5335,7 @@ static void open_db(ShellState *p, int openFlags){ zDbFilename); } } + globalDb = p->db; sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0); /* Reflect the use or absence of --unsafe-testing invocation. */