From 51d7869096bbeefb4f5365f29c78f47287e822df Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 20 Jul 2023 17:45:09 +0000 Subject: [PATCH 01/16] Experimental framework upon which to build a better JSON parse structure that supports cached of modified JSON. All of these changes are tentative and subject to change or removal. Incremental check-in. FossilOrigin-Name: e384163a4763c10b5838cbf28f9c4d8ea799bbbed6e890e8aa91b2bd1458646d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 26 ++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 140124f2f1..9bdb33ae9d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\simprovement\sto\sJSON\sparser\sperformance. -D 2023-07-19T17:24:36.452 +C Experimental\sframework\supon\swhich\sto\sbuild\sa\sbetter\sJSON\sparse\sstructure\sthat\nsupports\scached\sof\smodified\sJSON.\s\sAll\sof\sthese\schanges\sare\stentative\sand\nsubject\sto\schange\sor\sremoval.\s\sIncremental\scheck-in. +D 2023-07-20T17:45:09.052 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e48136fce64e5004b1b8be76624cc9a4ac9ff6ae630a97df187c0ea4b9692d1b +F src/json.c 888fcc121f7466bdf1518a8ae6adbfa7c4562438d046f1f21c942d625a208143 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P adb4d6b007cbe9d7c9670f5fc196443ebe0f3a89df1f3290ba6247fcf83fe5bd -R ab82eb710f468841d4023aaae186be2d +P 144c8ccf6e5bb2527dd98742f0d67e0a16c627e7c67f754ce8ed4c4fb5b8d8b6 +R 086ba0782e2494053078635aa92191a8 U drh -Z d9c18383c20eeea0feca70f3dd0775a9 +Z a6562dc595d378e643249c9e30c8bba6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4a7e995a3f..654058e2a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -144c8ccf6e5bb2527dd98742f0d67e0a16c627e7c67f754ce8ed4c4fb5b8d8b6 \ No newline at end of file +e384163a4763c10b5838cbf28f9c4d8ea799bbbed6e890e8aa91b2bd1458646d \ No newline at end of file diff --git a/src/json.c b/src/json.c index 176dcbfdb9..2b309ee32e 100644 --- a/src/json.c +++ b/src/json.c @@ -59,6 +59,7 @@ static const char jsonIsSpace[] = { typedef struct JsonString JsonString; typedef struct JsonNode JsonNode; typedef struct JsonParse JsonParse; +typedef struct JsonTask JsonTask; /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator @@ -74,6 +75,15 @@ struct JsonString { char zSpace[100]; /* Initial static space */ }; +/* A deferred cleanup task. A list of JsonTask objects might be +** run when the JsonParse object is destroyed. +*/ +struct JsonTask { + JsonTask *pJTNext; /* Next in a list */ + void (*xOp)(void*); /* Routine to run */ + void *pArg; /* Argument to xOp() */ +}; + /* JSON type values */ #define JSON_NULL 0 @@ -123,6 +133,7 @@ struct JsonNode { } u; }; + /* A completely parsed JSON string */ struct JsonParse { @@ -131,10 +142,13 @@ struct JsonParse { JsonNode *aNode; /* Array of nodes containing the parse */ const char *zJson; /* Original JSON string */ u32 *aUp; /* Index of parent of each node */ + JsonTask *pClean; /* Cleanup operations prior to freeing this object */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ + u8 nJPRef; /* Number of references to this object */ + u8 bIgnoreEdits; /* Ignore edit marks during search */ int nJson; /* Length of the zJson string in bytes */ u32 iErr; /* Error location in zJson[] */ u32 iHold; /* Replace cache line with the lowest iHold value */ @@ -531,6 +545,8 @@ static u32 jsonNodeSize(JsonNode *pNode){ ** delete the JsonParse object itself. */ static void jsonParseReset(JsonParse *pParse){ + assert( pParse->pClean==0 ); + assert( pParse->nJPRef<=1 ); sqlite3_free(pParse->aNode); pParse->aNode = 0; pParse->nNode = 0; @@ -543,6 +559,16 @@ static void jsonParseReset(JsonParse *pParse){ ** Free a JsonParse object that was obtained from sqlite3_malloc(). */ static void jsonParseFree(JsonParse *pParse){ + if( pParse->nJPRef>1 ){ + pParse->nJPRef--; + return; + } + while( pParse->pClean ){ + JsonTask *pTask = pParse->pClean; + pParse->pClean = pTask->pJTNext; + pTask->xOp(pTask->pArg); + sqlite3_free(pTask); + } jsonParseReset(pParse); sqlite3_free(pParse); } From c8f4fec2dea3c2d89eaa38bf5eb7dd0de8b02e37 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 11:09:53 +0000 Subject: [PATCH 02/16] More infrastructure changes towards improving JSON cache performance. Incremental check-in. FossilOrigin-Name: 1955e66cfc4614df97b8d68b0e662f309513d62dc8aeec71af5a54e66b79c707 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/json.c | 61 +++++++++++++++++++++++++++++---------------------- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 9bdb33ae9d..715018c361 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\sframework\supon\swhich\sto\sbuild\sa\sbetter\sJSON\sparse\sstructure\sthat\nsupports\scached\sof\smodified\sJSON.\s\sAll\sof\sthese\schanges\sare\stentative\sand\nsubject\sto\schange\sor\sremoval.\s\sIncremental\scheck-in. -D 2023-07-20T17:45:09.052 +C More\sinfrastructure\schanges\stowards\simproving\sJSON\scache\sperformance.\nIncremental\scheck-in. +D 2023-07-21T11:09:53.138 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 888fcc121f7466bdf1518a8ae6adbfa7c4562438d046f1f21c942d625a208143 +F src/json.c 3bdc8916ee1acf47f63373278b7913befea89fc8f4bfc0299ae9a2b34b35ee07 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 144c8ccf6e5bb2527dd98742f0d67e0a16c627e7c67f754ce8ed4c4fb5b8d8b6 -R 086ba0782e2494053078635aa92191a8 +P e384163a4763c10b5838cbf28f9c4d8ea799bbbed6e890e8aa91b2bd1458646d +R 35c799f493c52a97264947f70c69a12e U drh -Z a6562dc595d378e643249c9e30c8bba6 +Z 7b5e59d8c6f3a8847c09c081e8b2a5bc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 654058e2a7..b1ca498e22 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e384163a4763c10b5838cbf28f9c4d8ea799bbbed6e890e8aa91b2bd1458646d \ No newline at end of file +1955e66cfc4614df97b8d68b0e662f309513d62dc8aeec71af5a54e66b79c707 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 2b309ee32e..b7c94a8a06 100644 --- a/src/json.c +++ b/src/json.c @@ -72,6 +72,7 @@ struct JsonString { u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ u8 bErr; /* True if an error has been encountered */ + u8 bOrig; /* Ignore edits when rendering JSON */ char zSpace[100]; /* Initial static space */ }; @@ -148,7 +149,7 @@ struct JsonParse { u8 oom; /* Set to true if out of memory */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 nJPRef; /* Number of references to this object */ - u8 bIgnoreEdits; /* Ignore edit marks during search */ + u8 bOrig; /* Ignore edit marks during search */ int nJson; /* Length of the zJson string in bytes */ u32 iErr; /* Error location in zJson[] */ u32 iHold; /* Replace cache line with the lowest iHold value */ @@ -174,6 +175,7 @@ static void jsonZero(JsonString *p){ p->nAlloc = sizeof(p->zSpace); p->nUsed = 0; p->bStatic = 1; + p->bOrig = 0; } /* Initialize the JsonString object @@ -584,7 +586,7 @@ static void jsonRenderNode( sqlite3_value **aReplace /* Replacement values */ ){ assert( pNode!=0 ); - if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ + if( (pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH)) && !pOut->bOrig ){ if( (pNode->jnFlags & JNODE_REPLACE)!=0 && ALWAYS(aReplace!=0) ){ assert( pNode->eU==4 ); jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); @@ -650,13 +652,13 @@ static void jsonRenderNode( jsonAppendChar(pOut, '['); for(;;){ while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pOut->bOrig ){ jsonAppendSeparator(pOut); jsonRenderNode(&pNode[j], pOut, aReplace); } j += jsonNodeSize(&pNode[j]); } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( (pNode->jnFlags & JNODE_APPEND)==0 || pOut->bOrig ) break; assert( pNode->eU==2 ); pNode = &pNode[pNode->u.iAppend]; j = 1; @@ -669,7 +671,7 @@ static void jsonRenderNode( jsonAppendChar(pOut, '{'); for(;;){ while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pOut->bOrig ){ jsonAppendSeparator(pOut); jsonRenderNode(&pNode[j], pOut, aReplace); jsonAppendChar(pOut, ':'); @@ -677,7 +679,7 @@ static void jsonRenderNode( } j += 1 + jsonNodeSize(&pNode[j+1]); } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( (pNode->jnFlags & JNODE_APPEND)==0 || pOut->bOrig ) break; assert( pNode->eU==2 ); pNode = &pNode[pNode->u.iAppend]; j = 1; @@ -694,10 +696,12 @@ static void jsonRenderNode( static void jsonReturnJson( JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ + sqlite3_value **aReplace, /* Array of replacement values */ + int bOrig /* Ignore edits if true */ ){ JsonString s; jsonInit(&s, pCtx); + s.bOrig = bOrig!=0; jsonRenderNode(pNode, &s, aReplace); jsonResult(&s); sqlite3_result_subtype(pCtx, JSON_SUBTYPE); @@ -740,7 +744,8 @@ static u32 jsonHexToInt4(const char *z){ static void jsonReturn( JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ + sqlite3_value **aReplace, /* Array of replacement values */ + int bOrig /* Ignore edits if true */ ){ switch( pNode->eType ){ default: { @@ -887,7 +892,7 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - jsonReturnJson(pNode, pCtx, aReplace); + jsonReturnJson(pNode, pCtx, aReplace, bOrig); break; } } @@ -1797,7 +1802,7 @@ static JsonNode *jsonLookupStep( const char *zKey; JsonNode *pRoot = &pParse->aNode[iRoot]; if( zPath[0]==0 ) return pRoot; - if( pRoot->jnFlags & JNODE_REPLACE ) return 0; + if( (pRoot->jnFlags & JNODE_REPLACE)!=0 && !pParse->bOrig ) return 0; if( zPath[0]=='.' ){ if( pRoot->eType!=JSON_OBJECT ) return 0; zPath++; @@ -1830,7 +1835,7 @@ static JsonNode *jsonLookupStep( j++; j += jsonNodeSize(&pRoot[j]); } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + if( (pRoot->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break; assert( pRoot->eU==2 ); iRoot += pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; @@ -1839,6 +1844,7 @@ static JsonNode *jsonLookupStep( if( pApnd ){ u32 iStart, iLabel; JsonNode *pNode; + assert( !pParse->bOrig ); iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); zPath += i; @@ -1868,10 +1874,10 @@ static JsonNode *jsonLookupStep( if( pRoot->eType!=JSON_ARRAY ) return 0; for(;;){ while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->bOrig ) i++; j += jsonNodeSize(&pBase[j]); } - if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; + if( (pBase->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break; assert( pBase->eU==2 ); iBase += pBase->u.iAppend; pBase = &pParse->aNode[iBase]; @@ -1901,11 +1907,13 @@ static JsonNode *jsonLookupStep( zPath += j + 1; j = 1; for(;;){ - while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; + while( j<=pRoot->n + && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE) && !pParse->bOrig)) + ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->bOrig ) i--; j += jsonNodeSize(&pRoot[j]); } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + if( (pRoot->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break; assert( pRoot->eU==2 ); iRoot += pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; @@ -1917,6 +1925,7 @@ static JsonNode *jsonLookupStep( if( i==0 && pApnd ){ u32 iStart; JsonNode *pNode; + assert( !pParse->bOrig ); iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); if( pParse->oom ) return 0; @@ -2269,15 +2278,15 @@ static void jsonExtractFunc( } if( pNode ){ if( flags & JSON_JSON ){ - jsonReturnJson(pNode, ctx, 0); + jsonReturnJson(pNode, ctx, 0, 1); }else{ - jsonReturn(pNode, ctx, 0); + jsonReturn(pNode, ctx, 0, 1); sqlite3_result_subtype(ctx, 0); } } }else{ pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturn(pNode, ctx, 0); + if( p->nErr==0 && pNode ) jsonReturn(pNode, ctx, 0, 1); } }else{ /* Two or more PATH arguments results in a JSON array with each @@ -2405,7 +2414,7 @@ static void jsonPatchFunc( pResult = jsonMergePatch(&x, 0, y.aNode); assert( pResult!=0 || x.oom ); if( pResult ){ - jsonReturnJson(pResult, ctx, 0); + jsonReturnJson(pResult, ctx, 0, 0); }else{ sqlite3_result_error_nomem(ctx); } @@ -2482,7 +2491,7 @@ static void jsonRemoveFunc( if( pNode ) pNode->jnFlags |= JNODE_REMOVE; } if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(x.aNode, ctx, 0); + jsonReturnJson(x.aNode, ctx, 0, 0); } remove_done: jsonParseReset(&x); @@ -2527,7 +2536,7 @@ static void jsonReplaceFunc( assert( x.aNode[0].eU==4 ); sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); }else{ - jsonReturnJson(x.aNode, ctx, argv); + jsonReturnJson(x.aNode, ctx, argv, 0); } replace_err: jsonParseReset(&x); @@ -2586,7 +2595,7 @@ static void jsonSetFunc( assert( x.aNode[0].eU==4 ); sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); }else{ - jsonReturnJson(x.aNode, ctx, argv); + jsonReturnJson(x.aNode, ctx, argv, 0); } jsonSetDone: jsonParseReset(&x); @@ -3095,7 +3104,7 @@ static int jsonEachColumn( case JEACH_KEY: { if( p->i==0 ) break; if( p->eType==JSON_OBJECT ){ - jsonReturn(pThis, ctx, 0); + jsonReturn(pThis, ctx, 0, 0); }else if( p->eType==JSON_ARRAY ){ u32 iKey; if( p->bRecursive ){ @@ -3111,7 +3120,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(pThis, ctx, 0); + jsonReturn(pThis, ctx, 0, 0); break; } case JEACH_TYPE: { @@ -3122,7 +3131,7 @@ static int jsonEachColumn( case JEACH_ATOM: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(pThis, ctx, 0); + jsonReturn(pThis, ctx, 0, 0); break; } case JEACH_ID: { From 9125d5ab6c043dca59ac5ff10f23558820701a01 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 24 Jul 2023 17:59:25 +0000 Subject: [PATCH 03/16] Incremental progress toward improved caching of parsed JSON. FossilOrigin-Name: f2c063884685a79d5a787590447c292f51e898a98c9508159c788f505227ba85 --- manifest | 12 +- manifest.uuid | 2 +- src/json.c | 334 +++++++++++++++++++++++++++++++++++++------------- 3 files changed, 256 insertions(+), 92 deletions(-) diff --git a/manifest b/manifest index 552d7e205e..e99cb3d132 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\senhancements\sinto\sthe\sjson-opt\sbranch. -D 2023-07-24T12:37:23.834 +C Incremental\sprogress\stoward\simproved\scaching\sof\sparsed\sJSON. +D 2023-07-24T17:59:25.604 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c d34d10ea1520bb71ee69c500deb4a20847a57b4c59c1ecb807c935f53c870885 +F src/json.c 3584f5fb0ff48009ebabb34704328c2086d924fc2403fdf0e48afce5b9f22777 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1955e66cfc4614df97b8d68b0e662f309513d62dc8aeec71af5a54e66b79c707 c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3 -R 2ab9518ef262a1aca9fe7f6a2ac934e0 +P 00bfc4918be49ac74a3e7851c88ab7ec226e6a37853f8ad4c77f758751960456 +R 831d8993ebdb0e60bc4ab13513e5d282 U drh -Z 40e6676e6b6212e46c825865d3333a87 +Z 8cedb8d1881aa5ce4f4f5c5fea73c221 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 37810abc34..5be0ea0fd7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00bfc4918be49ac74a3e7851c88ab7ec226e6a37853f8ad4c77f758751960456 \ No newline at end of file +f2c063884685a79d5a787590447c292f51e898a98c9508159c788f505227ba85 \ No newline at end of file diff --git a/src/json.c b/src/json.c index bf7c2be3ae..97ebb6af65 100644 --- a/src/json.c +++ b/src/json.c @@ -72,7 +72,6 @@ struct JsonString { u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ u8 bErr; /* True if an error has been encountered */ - u8 bOrig; /* Ignore edits when rendering JSON */ char zSpace[100]; /* Initial static space */ }; @@ -87,14 +86,16 @@ struct JsonTask { /* JSON type values */ -#define JSON_NULL 0 -#define JSON_TRUE 1 -#define JSON_FALSE 2 -#define JSON_INT 3 -#define JSON_REAL 4 -#define JSON_STRING 5 -#define JSON_ARRAY 6 -#define JSON_OBJECT 7 +#define JSON_SUBST 0 /* Special edit node. Uses u.iPrev */ +#define JSON_PATCH 1 /* Special edit node. Uses u.pPatch */ +#define JSON_NULL 2 +#define JSON_TRUE 3 +#define JSON_FALSE 4 +#define JSON_INT 5 +#define JSON_REAL 6 +#define JSON_STRING 7 +#define JSON_ARRAY 8 +#define JSON_OBJECT 9 /* The "subtype" set for JSON values */ #define JSON_SUBTYPE 74 /* Ascii for "J" */ @@ -103,6 +104,7 @@ struct JsonTask { ** Names of the various JSON types: */ static const char * const jsonType[] = { + "subst", "patch", "null", "true", "false", "integer", "real", "text", "array", "object" }; @@ -124,13 +126,15 @@ struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 jnFlags; /* JNODE flags */ u8 eU; /* Which union element to use */ - u32 n; /* Bytes of content, or number of sub-nodes */ + u32 n; /* Bytes of content for INT, REAL or STRING + ** Number of sub-nodes for ARRAY and OBJECT + ** Node that SUBST applies to */ union { const char *zJContent; /* 1: Content for INT, REAL, and STRING */ u32 iAppend; /* 2: More terms for ARRAY and OBJECT */ u32 iKey; /* 3: Key for ARRAY objects in json_tree() */ - u32 iReplace; /* 4: Replacement content for JNODE_REPLACE */ JsonNode *pPatch; /* 5: Node chain of patch for JNODE_PATCH */ + u32 iPrev; /* 6: Previous SUBST node, or 0 */ } u; }; @@ -149,9 +153,9 @@ struct JsonParse { u8 oom; /* Set to true if out of memory */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 nJPRef; /* Number of references to this object */ - u8 bOrig; /* Ignore edit marks during search */ int nJson; /* Length of the zJson string in bytes */ u32 iErr; /* Error location in zJson[] */ + u32 iSubst; /* Last known JSON_SUBST node */ u32 iHold; /* Replace cache line with the lowest iHold value */ }; @@ -175,7 +179,6 @@ static void jsonZero(JsonString *p){ p->nAlloc = sizeof(p->zSpace); p->nUsed = 0; p->bStatic = 1; - p->bOrig = 0; } /* Initialize the JsonString object @@ -548,7 +551,12 @@ static u32 jsonNodeSize(JsonNode *pNode){ ** delete the JsonParse object itself. */ static void jsonParseReset(JsonParse *pParse){ - assert( pParse->pClean==0 ); + while( pParse->pClean ){ + JsonTask *pTask = pParse->pClean; + pParse->pClean = pTask->pJTNext; + pTask->xOp(pTask->pArg); + sqlite3_free(pTask); + } assert( pParse->nJPRef<=1 ); sqlite3_free(pParse->aNode); pParse->aNode = 0; @@ -566,35 +574,63 @@ static void jsonParseFree(JsonParse *pParse){ pParse->nJPRef--; return; } - while( pParse->pClean ){ - JsonTask *pTask = pParse->pClean; - pParse->pClean = pTask->pJTNext; - pTask->xOp(pTask->pArg); - sqlite3_free(pTask); - } jsonParseReset(pParse); sqlite3_free(pParse); } +/* +** Add a cleanup task to the JsonParse object. +** +** If an OOM occurs, the cleanup operation happens immediately +** and this function returns SQLITE_NOMEM. +*/ +static int jsonParseAddCleanup( + JsonParse *pParse, /* Add the cleanup task to this parser */ + void(*xOp)(void*), /* The cleanup task */ + void *pArg /* Argument to the cleanup */ +){ + JsonTask *pTask = sqlite3_malloc64( sizeof(*pTask) ); + if( pTask==0 ){ + pParse->oom = 1; + xOp(pArg); + return SQLITE_ERROR; + } + pTask->pJTNext = pParse->pClean; + pParse->pClean = pTask; + pTask->xOp = xOp; + pTask->pArg = pArg; + return SQLITE_OK; +} + /* ** Convert the JsonNode pNode into a pure JSON string and ** append to pOut. Subsubstructure is also included. Return ** the number of JsonNode objects that are encoded. */ static void jsonRenderNode( + JsonParse *pParse, /* the complete parse of the JSON */ JsonNode *pNode, /* The node to render */ - JsonString *pOut, /* Write JSON here */ - sqlite3_value **aReplace /* Replacement values */ + JsonString *pOut /* Write JSON here */ ){ assert( pNode!=0 ); - if( (pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH)) && !pOut->bOrig ){ - if( (pNode->jnFlags & JNODE_REPLACE)!=0 && ALWAYS(aReplace!=0) ){ - assert( pNode->eU==4 ); - jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); - return; + while( (pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH)) ){ + if( (pNode->jnFlags & JNODE_REPLACE)!=0 ){ + u32 idx = (u32)(pNode - pParse->aNode); + u32 i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( inNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==6 ); + assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ + pNode = &pParse->aNode[i+1]; + break; + } + i = pParse->aNode[i].u.iPrev; + } + }else{ + pNode = pNode->u.pPatch; } - assert( pNode->eU==5 ); - pNode = pNode->u.pPatch; } switch( pNode->eType ){ default: { @@ -653,13 +689,13 @@ static void jsonRenderNode( jsonAppendChar(pOut, '['); for(;;){ while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pOut->bOrig ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); + jsonRenderNode(pParse, &pNode[j], pOut); } j += jsonNodeSize(&pNode[j]); } - if( (pNode->jnFlags & JNODE_APPEND)==0 || pOut->bOrig ) break; + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; assert( pNode->eU==2 ); pNode = &pNode[pNode->u.iAppend]; j = 1; @@ -672,15 +708,15 @@ static void jsonRenderNode( jsonAppendChar(pOut, '{'); for(;;){ while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pOut->bOrig ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); + jsonRenderNode(pParse, &pNode[j], pOut); jsonAppendChar(pOut, ':'); - jsonRenderNode(&pNode[j+1], pOut, aReplace); + jsonRenderNode(pParse, &pNode[j+1], pOut); } j += 1 + jsonNodeSize(&pNode[j+1]); } - if( (pNode->jnFlags & JNODE_APPEND)==0 || pOut->bOrig ) break; + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; assert( pNode->eU==2 ); pNode = &pNode[pNode->u.iAppend]; j = 1; @@ -695,15 +731,13 @@ static void jsonRenderNode( ** Return a JsonNode and all its descendants as a JSON string. */ static void jsonReturnJson( + JsonParse *pParse, /* The complete JSON */ JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace, /* Array of replacement values */ - int bOrig /* Ignore edits if true */ + sqlite3_context *pCtx /* Return value for this function */ ){ JsonString s; jsonInit(&s, pCtx); - s.bOrig = bOrig!=0; - jsonRenderNode(pNode, &s, aReplace); + jsonRenderNode(pParse, pNode, &s); jsonResult(&s); sqlite3_result_subtype(pCtx, JSON_SUBTYPE); } @@ -743,10 +777,9 @@ static u32 jsonHexToInt4(const char *z){ ** Make the JsonNode the return value of the function. */ static void jsonReturn( + JsonParse *pParse, /* Complete JSON parse tree */ JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace, /* Array of replacement values */ - int bOrig /* Ignore edits if true */ + sqlite3_context *pCtx /* Return value for this function */ ){ switch( pNode->eType ){ default: { @@ -893,7 +926,7 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - jsonReturnJson(pNode, pCtx, aReplace, bOrig); + jsonReturnJson(pParse, pNode, pCtx); break; } } @@ -962,6 +995,22 @@ static int jsonParseAddNode( return pParse->nNode++; } +/* +** Add a new JSON_SUBST node. The node immediately following +** this new node will be the substitute content for iNode. +*/ +static int jsonParseAddSubstNode( + JsonParse *pParse, /* Add the JSON_SUBST here */ + u32 iNode /* References this node */ +){ + int idx = jsonParseAddNode(pParse, JSON_SUBST, iNode, 0); + if( idx<=0 ) return idx; + pParse->aNode[idx].eU = 6; + pParse->aNode[idx].u.iPrev = pParse->iSubst; + pParse->iSubst = idx; + return idx; +} + /* ** Return true if z[] begins with 2 (or more) hexadecimal digits */ @@ -1749,6 +1798,7 @@ static JsonParse *jsonParseCached( sqlite3_free(p); return 0; } + p->nJPRef = 1; p->nJson = nJson; p->iHold = iMaxHold+1; sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, @@ -1803,7 +1853,6 @@ static JsonNode *jsonLookupStep( const char *zKey; JsonNode *pRoot = &pParse->aNode[iRoot]; if( zPath[0]==0 ) return pRoot; - if( (pRoot->jnFlags & JNODE_REPLACE)!=0 && !pParse->bOrig ) return 0; if( zPath[0]=='.' ){ if( pRoot->eType!=JSON_OBJECT ) return 0; zPath++; @@ -1836,7 +1885,7 @@ static JsonNode *jsonLookupStep( j++; j += jsonNodeSize(&pRoot[j]); } - if( (pRoot->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break; + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; assert( pRoot->eU==2 ); iRoot += pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; @@ -1845,7 +1894,6 @@ static JsonNode *jsonLookupStep( if( pApnd ){ u32 iStart, iLabel; JsonNode *pNode; - assert( !pParse->bOrig ); iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); zPath += i; @@ -1875,10 +1923,10 @@ static JsonNode *jsonLookupStep( if( pRoot->eType!=JSON_ARRAY ) return 0; for(;;){ while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->bOrig ) i++; + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; j += jsonNodeSize(&pBase[j]); } - if( (pBase->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break; + if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; assert( pBase->eU==2 ); iBase += pBase->u.iAppend; pBase = &pParse->aNode[iBase]; @@ -1908,13 +1956,11 @@ static JsonNode *jsonLookupStep( zPath += j + 1; j = 1; for(;;){ - while( j<=pRoot->n - && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE) && !pParse->bOrig)) - ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->bOrig ) i--; + while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; j += jsonNodeSize(&pRoot[j]); } - if( (pRoot->jnFlags & JNODE_APPEND)==0 || pParse->bOrig ) break; + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; assert( pRoot->eU==2 ); iRoot += pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; @@ -1926,7 +1972,6 @@ static JsonNode *jsonLookupStep( if( i==0 && pApnd ){ u32 iStart; JsonNode *pNode; - assert( !pParse->bOrig ); iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); if( pParse->oom ) return 0; @@ -2060,6 +2105,60 @@ static void jsonRemoveAllNulls(JsonNode *pNode){ ** SQL functions used for testing and debugging ****************************************************************************/ +#if 0 /* One for debugging. zero normally */ +/* +** Print N node entries. +*/ +static void jsonDebugPrintNodeEntries( + JsonNode *aNode, /* First node entry to print */ + int N, /* Number of node entries to print */ + int ofst, /* Node number for p */ + int nDent /* Indent by this many spaces */ +){ + int i; + for(i=0; i0 ) printf("%*s", nDent, ""); + if( (aNode[i].jnFlags & ~JNODE_LABEL)!=0 ){ + printf("node %4u: %-7s %02x n=%-5d", + i+ofst, zType, aNode[i].jnFlags, aNode[i].n); + }else{ + printf("node %4u: %-7s n=%-5d", + i+ofst, zType, aNode[i].n); + } + switch( aNode[i].eU ){ + case 1: printf(" zJContent=[%.*s]\n", + aNode[i].n, aNode[i].u.zJContent); break; + case 2: printf(" iAppend=%u\n", aNode[i].u.iAppend); break; + case 3: printf(" iKey=%u\n", aNode[i].u.iKey); break; + case 5: { + JsonNode *pX = aNode[i].u.pPatch; + printf(" pPatch=...\n"); + jsonDebugPrintNodeEntries(pX, jsonNodeSize(pX), 0, nDent+3); + break; + } + case 6: printf(" iPrev=%u\n", aNode[i].u.iPrev); break; + default: printf("\n"); + } + } +} +static void jsonDebugPrintParse(JsonParse *p){ + jsonDebugPrintNodeEntries(p->aNode, p->nNode, 0, 0); +} +static void jsonDebugPrintNode(JsonNode *pNode){ + jsonDebugPrintNodeEntries(pNode, jsonNodeSize(pNode), 0, 0); +} +#else + /* The usual case */ +# define jsonDebugPrintNode(X) +# define jsonDebugPrintParse(X) +#endif + #ifdef SQLITE_DEBUG /* ** The json_parse(JSON) function returns a string which describes @@ -2279,15 +2378,15 @@ static void jsonExtractFunc( } if( pNode ){ if( flags & JSON_JSON ){ - jsonReturnJson(pNode, ctx, 0, 1); + jsonReturnJson(p, pNode, ctx); }else{ - jsonReturn(pNode, ctx, 0, 1); + jsonReturn(p, pNode, ctx); sqlite3_result_subtype(ctx, 0); } } }else{ pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturn(pNode, ctx, 0, 1); + if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx); } }else{ /* Two or more PATH arguments results in a JSON array with each @@ -2301,7 +2400,7 @@ static void jsonExtractFunc( if( p->nErr ) break; jsonAppendSeparator(&jx); if( pNode ){ - jsonRenderNode(pNode, &jx, 0); + jsonRenderNode(p, pNode, &jx); }else{ jsonAppendRawNZ(&jx, "null", 4); } @@ -2415,7 +2514,8 @@ static void jsonPatchFunc( pResult = jsonMergePatch(&x, 0, y.aNode); assert( pResult!=0 || x.oom ); if( pResult ){ - jsonReturnJson(pResult, ctx, 0, 0); + jsonDebugPrintNode(pResult); + jsonReturnJson(&x, pResult, ctx); }else{ sqlite3_result_error_nomem(ctx); } @@ -2492,12 +2592,89 @@ static void jsonRemoveFunc( if( pNode ) pNode->jnFlags |= JNODE_REMOVE; } if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(x.aNode, ctx, 0, 0); + jsonReturnJson(&x, x.aNode, ctx); } remove_done: + jsonDebugPrintParse(&x); jsonParseReset(&x); } +/* +** Substitute the value at iNode with the pValue parameter. +*/ +static void jsonReplaceNode( + sqlite3_context *pCtx, + JsonParse *p, + int iNode, + sqlite3_value *pValue +){ + int idx = jsonParseAddSubstNode(p, iNode); + if( idx<=0 ){ + assert( p->oom ); + return; + } + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonParseAddNode(p, JSON_NULL, 0, 0); + break; + } + case SQLITE_FLOAT: { + char *z = sqlite3_mprintf("%!0.15g", sqlite3_value_double(pValue)); + int n; + if( z==0 ){ + p->oom = 1; + break; + } + n = sqlite3Strlen30(z); + jsonParseAddNode(p, JSON_REAL, n, z); + jsonParseAddCleanup(p, sqlite3_free, z); + break; + } + case SQLITE_INTEGER: { + char *z = sqlite3_mprintf("%lld", sqlite3_value_int64(pValue)); + int n; + if( z==0 ){ + p->oom = 1; + break; + } + n = sqlite3Strlen30(z); + jsonParseAddNode(p, JSON_INT, n, z); + jsonParseAddCleanup(p, sqlite3_free, z); + + break; + } + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + if( z==0 ){ + p->oom = 1; + break; + } + if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){ + int k = jsonParseAddNode(p, JSON_STRING, n, z); + if( k>0 ) p->aNode[k].jnFlags |= JNODE_RAW; + jsonParseAddCleanup(p, sqlite3_free, sqlite3DbStrDup(0,z)); + }else{ + int k= jsonParseAddNode(p, JSON_PATCH, 0, 0); + if( k>0 ){ + JsonParse *pPatch = jsonParseCached(pCtx, &pValue, pCtx); + if( pPatch==0 ){ + p->oom = 1; + break; + } + assert( pPatch->nJPRef>=1 ); + pPatch->nJPRef++; + p->aNode[k].jnFlags |= JNODE_PATCH; + p->aNode[k].eU = 5; + p->aNode[k].u.pPatch = pPatch->aNode; + jsonParseAddCleanup(p, (void(*)(void*))jsonParseFree, pPatch); + } + } + break; + } + } +} + /* ** json_replace(JSON, PATH, VALUE, ...) ** @@ -2526,20 +2703,13 @@ static void jsonReplaceFunc( pNode = jsonLookup(&x, zPath, 0, ctx); if( x.nErr ) goto replace_err; if( pNode ){ - assert( pNode->eU==0 || pNode->eU==1 || pNode->eU==4 ); - testcase( pNode->eU!=0 && pNode->eU!=1 ); pNode->jnFlags |= (u8)JNODE_REPLACE; - VVA( pNode->eU = 4 ); - pNode->u.iReplace = i + 1; + jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]); } } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - assert( x.aNode[0].eU==4 ); - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv, 0); - } + jsonReturnJson(&x, x.aNode, ctx); replace_err: + jsonDebugPrintParse(&x); jsonParseReset(&x); } @@ -2585,19 +2755,13 @@ static void jsonSetFunc( }else if( x.nErr ){ goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ - testcase( pNode->eU!=0 && pNode->eU!=1 ); - assert( pNode->eU!=3 && pNode->eU!=5 ); - VVA( pNode->eU = 4 ); pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->u.iReplace = i + 1; + jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]); } } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - assert( x.aNode[0].eU==4 ); - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv, 0); - } + jsonDebugPrintParse(&x); + jsonReturnJson(&x, x.aNode, ctx); + jsonSetDone: jsonParseReset(&x); } @@ -3105,7 +3269,7 @@ static int jsonEachColumn( case JEACH_KEY: { if( p->i==0 ) break; if( p->eType==JSON_OBJECT ){ - jsonReturn(pThis, ctx, 0, 0); + jsonReturn(&p->sParse, pThis, ctx); }else if( p->eType==JSON_ARRAY ){ u32 iKey; if( p->bRecursive ){ @@ -3121,7 +3285,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(pThis, ctx, 0, 0); + jsonReturn(&p->sParse, pThis, ctx); break; } case JEACH_TYPE: { @@ -3132,7 +3296,7 @@ static int jsonEachColumn( case JEACH_ATOM: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(pThis, ctx, 0, 0); + jsonReturn(&p->sParse, pThis, ctx); break; } case JEACH_ID: { From 27553579c0c15fa12d07f85da6ff12ffc1425b35 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 24 Jul 2023 22:34:26 +0000 Subject: [PATCH 04/16] Rework the JSON parse structure to facilitate better caching. Passes all tests. FossilOrigin-Name: ecdcb1ded76e9a0591bf7a2009679f49fc3aa639d3cc12406c6d29243ed8e1c5 --- manifest | 12 +-- manifest.uuid | 2 +- src/json.c | 224 ++++++++++++++++++++++++++++---------------------- 3 files changed, 135 insertions(+), 103 deletions(-) diff --git a/manifest b/manifest index e99cb3d132..6a75e96be4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Incremental\sprogress\stoward\simproved\scaching\sof\sparsed\sJSON. -D 2023-07-24T17:59:25.604 +C Rework\sthe\sJSON\sparse\sstructure\sto\sfacilitate\sbetter\scaching.\s\sPasses\sall\ntests. +D 2023-07-24T22:34:26.093 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 3584f5fb0ff48009ebabb34704328c2086d924fc2403fdf0e48afce5b9f22777 +F src/json.c e743eb83ad6d581bdfd355aee1d0f2c3025e955ec1799706ec4542b46e3af7f2 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 00bfc4918be49ac74a3e7851c88ab7ec226e6a37853f8ad4c77f758751960456 -R 831d8993ebdb0e60bc4ab13513e5d282 +P f2c063884685a79d5a787590447c292f51e898a98c9508159c788f505227ba85 +R fee4083d53f0bb74692c3aad54ce5646 U drh -Z 8cedb8d1881aa5ce4f4f5c5fea73c221 +Z 5a89ce2654bac729f780024f319554e3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5be0ea0fd7..0cc1f0a629 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2c063884685a79d5a787590447c292f51e898a98c9508159c788f505227ba85 \ No newline at end of file +ecdcb1ded76e9a0591bf7a2009679f49fc3aa639d3cc12406c6d29243ed8e1c5 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 97ebb6af65..b109a8759d 100644 --- a/src/json.c +++ b/src/json.c @@ -87,15 +87,14 @@ struct JsonTask { /* JSON type values */ #define JSON_SUBST 0 /* Special edit node. Uses u.iPrev */ -#define JSON_PATCH 1 /* Special edit node. Uses u.pPatch */ -#define JSON_NULL 2 -#define JSON_TRUE 3 -#define JSON_FALSE 4 -#define JSON_INT 5 -#define JSON_REAL 6 -#define JSON_STRING 7 -#define JSON_ARRAY 8 -#define JSON_OBJECT 9 +#define JSON_NULL 1 +#define JSON_TRUE 2 +#define JSON_FALSE 3 +#define JSON_INT 4 +#define JSON_REAL 5 +#define JSON_STRING 6 +#define JSON_ARRAY 7 +#define JSON_OBJECT 8 /* The "subtype" set for JSON values */ #define JSON_SUBTYPE 74 /* Ascii for "J" */ @@ -104,7 +103,7 @@ struct JsonTask { ** Names of the various JSON types: */ static const char * const jsonType[] = { - "subst", "patch", + "subst", "null", "true", "false", "integer", "real", "text", "array", "object" }; @@ -113,8 +112,8 @@ static const char * const jsonType[] = { #define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ #define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ #define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ -#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ +#define JNODE_REPLACE 0x08 /* Target of a JSON_SUBST node */ + /* 0x10 Available for reuse */ #define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ #define JNODE_LABEL 0x40 /* Is a label of an object */ #define JNODE_JSON5 0x80 /* Node contains JSON5 enhancements */ @@ -133,8 +132,7 @@ struct JsonNode { const char *zJContent; /* 1: Content for INT, REAL, and STRING */ u32 iAppend; /* 2: More terms for ARRAY and OBJECT */ u32 iKey; /* 3: Key for ARRAY objects in json_tree() */ - JsonNode *pPatch; /* 5: Node chain of patch for JNODE_PATCH */ - u32 iPrev; /* 6: Previous SUBST node, or 0 */ + u32 iPrev; /* 4: Previous SUBST node, or 0 */ } u; }; @@ -613,23 +611,19 @@ static void jsonRenderNode( JsonString *pOut /* Write JSON here */ ){ assert( pNode!=0 ); - while( (pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH)) ){ - if( (pNode->jnFlags & JNODE_REPLACE)!=0 ){ - u32 idx = (u32)(pNode - pParse->aNode); - u32 i = pParse->iSubst; - while( 1 /*exit-by-break*/ ){ - assert( inNode ); - assert( pParse->aNode[i].eType==JSON_SUBST ); - assert( pParse->aNode[i].eU==6 ); - assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ - pNode = &pParse->aNode[i+1]; - break; - } - i = pParse->aNode[i].u.iPrev; + while( (pNode->jnFlags & JNODE_REPLACE)!=0 ){ + u32 idx = (u32)(pNode - pParse->aNode); + u32 i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( inNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==4 ); + assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ + pNode = &pParse->aNode[i+1]; + break; } - }else{ - pNode = pNode->u.pPatch; + i = pParse->aNode[i].u.iPrev; } } switch( pNode->eType ){ @@ -697,7 +691,7 @@ static void jsonRenderNode( } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; assert( pNode->eU==2 ); - pNode = &pNode[pNode->u.iAppend]; + pNode = &pParse->aNode[pNode->u.iAppend]; j = 1; } jsonAppendChar(pOut, ']'); @@ -718,7 +712,7 @@ static void jsonRenderNode( } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; assert( pNode->eU==2 ); - pNode = &pNode[pNode->u.iAppend]; + pNode = &pParse->aNode[pNode->u.iAppend]; j = 1; } jsonAppendChar(pOut, '}'); @@ -736,6 +730,10 @@ static void jsonReturnJson( sqlite3_context *pCtx /* Return value for this function */ ){ JsonString s; + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } jsonInit(&s, pCtx); jsonRenderNode(pParse, pNode, &s); jsonResult(&s); @@ -948,6 +946,12 @@ static int jsonParseAddNode(JsonParse*,u32,u32,const char*); #endif +/* +** Add a single node to pParse->aNode after first expanding the +** size of the aNode array. Return the index of the new node. +** +** If an OOM error occurs, set pParse->oom and return -1. +*/ static JSON_NOINLINE int jsonParseAddNodeExpand( JsonParse *pParse, /* Append the node to this object */ u32 eType, /* Node type */ @@ -964,7 +968,7 @@ static JSON_NOINLINE int jsonParseAddNodeExpand( pParse->oom = 1; return -1; } - pParse->nAlloc = nNew; + pParse->nAlloc = sqlite3_msize(pNew)/sizeof(JsonNode); pParse->aNode = pNew; assert( pParse->nNodenAlloc ); return jsonParseAddNode(pParse, eType, n, zContent); @@ -995,6 +999,31 @@ static int jsonParseAddNode( return pParse->nNode++; } +/* +** Add an array of new nodes to the current pParse->aNode array. +** Return the index of the first node added. +** +** If an OOM error occurs, set pParse->oom. +*/ +static void jsonParseAddNodeArray( + JsonParse *pParse, /* Append the node to this object */ + JsonNode *aNode, /* Array of nodes to add */ + u32 nNode /* Number of elements in aNew */ +){ + if( pParse->nNode + nNode > pParse->nAlloc ){ + u32 nNew = pParse->nNode + nNode; + JsonNode *aNew = sqlite3_realloc64(pParse->aNode, nNew*sizeof(JsonNode)); + if( aNew==0 ){ + pParse->oom = 1; + return; + } + pParse->nAlloc = sqlite3_msize(aNew)/sizeof(JsonNode); + pParse->aNode = aNew; + } + memcpy(&pParse->aNode[pParse->nNode], aNode, nNode*sizeof(JsonNode)); + pParse->nNode += nNode; +} + /* ** Add a new JSON_SUBST node. The node immediately following ** this new node will be the substitute content for iNode. @@ -1004,8 +1033,9 @@ static int jsonParseAddSubstNode( u32 iNode /* References this node */ ){ int idx = jsonParseAddNode(pParse, JSON_SUBST, iNode, 0); - if( idx<=0 ) return idx; - pParse->aNode[idx].eU = 6; + if( pParse->oom ) return -1; + pParse->aNode[iNode].jnFlags |= JNODE_REPLACE; + pParse->aNode[idx].eU = 4; pParse->aNode[idx].u.iPrev = pParse->iSubst; pParse->iSubst = idx; return idx; @@ -1852,6 +1882,22 @@ static JsonNode *jsonLookupStep( u32 i, j, nKey; const char *zKey; JsonNode *pRoot = &pParse->aNode[iRoot]; + while( (pRoot->jnFlags & JNODE_REPLACE)!=0 ){ + u32 idx = (u32)(pRoot - pParse->aNode); + u32 i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( inNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==4 ); + assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ + pRoot = &pParse->aNode[i+1]; + iRoot = i+1; + break; + } + i = pParse->aNode[i].u.iPrev; + } + } if( zPath[0]==0 ) return pRoot; if( zPath[0]=='.' ){ if( pRoot->eType!=JSON_OBJECT ) return 0; @@ -1887,7 +1933,7 @@ static JsonNode *jsonLookupStep( } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; assert( pRoot->eU==2 ); - iRoot += pRoot->u.iAppend; + iRoot = pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; j = 1; } @@ -1902,7 +1948,7 @@ static JsonNode *jsonLookupStep( if( pNode ){ pRoot = &pParse->aNode[iRoot]; assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart - iRoot; + pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; VVA( pRoot->eU = 2 ); pParse->aNode[iLabel].jnFlags |= JNODE_RAW; @@ -1928,7 +1974,7 @@ static JsonNode *jsonLookupStep( } if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; assert( pBase->eU==2 ); - iBase += pBase->u.iAppend; + iBase = pBase->u.iAppend; pBase = &pParse->aNode[iBase]; j = 1; } @@ -1962,7 +2008,7 @@ static JsonNode *jsonLookupStep( } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; assert( pRoot->eU==2 ); - iRoot += pRoot->u.iAppend; + iRoot = pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; j = 1; } @@ -1978,7 +2024,7 @@ static JsonNode *jsonLookupStep( if( pNode ){ pRoot = &pParse->aNode[iRoot]; assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart - iRoot; + pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; VVA( pRoot->eU = 2 ); } @@ -2111,9 +2157,7 @@ static void jsonRemoveAllNulls(JsonNode *pNode){ */ static void jsonDebugPrintNodeEntries( JsonNode *aNode, /* First node entry to print */ - int N, /* Number of node entries to print */ - int ofst, /* Node number for p */ - int nDent /* Indent by this many spaces */ + int N /* Number of node entries to print */ ){ int i; for(i=0; i0 ) printf("%*s", nDent, ""); if( (aNode[i].jnFlags & ~JNODE_LABEL)!=0 ){ printf("node %4u: %-7s %02x n=%-5d", - i+ofst, zType, aNode[i].jnFlags, aNode[i].n); + i, zType, aNode[i].jnFlags, aNode[i].n); }else{ printf("node %4u: %-7s n=%-5d", - i+ofst, zType, aNode[i].n); + i, zType, aNode[i].n); } switch( aNode[i].eU ){ case 1: printf(" zJContent=[%.*s]\n", aNode[i].n, aNode[i].u.zJContent); break; case 2: printf(" iAppend=%u\n", aNode[i].u.iAppend); break; case 3: printf(" iKey=%u\n", aNode[i].u.iKey); break; - case 5: { - JsonNode *pX = aNode[i].u.pPatch; - printf(" pPatch=...\n"); - jsonDebugPrintNodeEntries(pX, jsonNodeSize(pX), 0, nDent+3); - break; - } - case 6: printf(" iPrev=%u\n", aNode[i].u.iPrev); break; + case 4: printf(" iPrev=%u\n", aNode[i].u.iPrev); break; default: printf("\n"); } } } static void jsonDebugPrintParse(JsonParse *p){ - jsonDebugPrintNodeEntries(p->aNode, p->nNode, 0, 0); + jsonDebugPrintNodeEntries(p->aNode, p->nNode); } static void jsonDebugPrintNode(JsonNode *pNode){ - jsonDebugPrintNodeEntries(pNode, jsonNodeSize(pNode), 0, 0); + jsonDebugPrintNodeEntries(pNode, jsonNodeSize(pNode)); } #else /* The usual case */ @@ -2447,45 +2484,38 @@ static JsonNode *jsonMergePatch( assert( pTarget[j].eType==JSON_STRING ); assert( pTarget[j].jnFlags & JNODE_LABEL ); if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){ - if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ) break; if( pPatch[i+1].eType==JSON_NULL ){ pTarget[j+1].jnFlags |= JNODE_REMOVE; }else{ JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); if( pNew==0 ) return 0; - pTarget = &pParse->aNode[iTarget]; - if( pNew!=&pTarget[j+1] ){ - assert( pTarget[j+1].eU==0 - || pTarget[j+1].eU==1 - || pTarget[j+1].eU==2 ); - testcase( pTarget[j+1].eU==1 ); - testcase( pTarget[j+1].eU==2 ); - VVA( pTarget[j+1].eU = 5 ); - pTarget[j+1].u.pPatch = pNew; - pTarget[j+1].jnFlags |= JNODE_PATCH; + if( pNew!=&pParse->aNode[iTarget+j+1] ){ + jsonParseAddSubstNode(pParse, iTarget+j+1); + jsonParseAddNodeArray(pParse, pNew, jsonNodeSize(pNew)); } + pTarget = &pParse->aNode[iTarget]; } break; } } if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ - int iStart, iPatch; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + int iStart; + JsonNode *pApnd; + u32 nApnd; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + pApnd = &pPatch[i+1]; + if( pApnd->eType==JSON_OBJECT ) jsonRemoveAllNulls(pApnd); + nApnd = jsonNodeSize(pApnd); + jsonParseAddNodeArray(pParse, pApnd, jsonNodeSize(pApnd)); if( pParse->oom ) return 0; - jsonRemoveAllNulls(pPatch); - pTarget = &pParse->aNode[iTarget]; - assert( pParse->aNode[iRoot].eU==0 || pParse->aNode[iRoot].eU==2 ); - testcase( pParse->aNode[iRoot].eU==2 ); + pParse->aNode[iStart].n = 1+nApnd; pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; + pParse->aNode[iRoot].u.iAppend = iStart; VVA( pParse->aNode[iRoot].eU = 2 ); - pParse->aNode[iRoot].u.iAppend = iStart - iRoot; iRoot = iStart; - assert( pParse->aNode[iPatch].eU==0 ); - VVA( pParse->aNode[iPatch].eU = 5 ); - pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; - pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + pTarget = &pParse->aNode[iTarget]; } } return pTarget; @@ -2513,7 +2543,8 @@ static void jsonPatchFunc( } pResult = jsonMergePatch(&x, 0, y.aNode); assert( pResult!=0 || x.oom ); - if( pResult ){ + if( pResult && x.oom==0 ){ + jsonDebugPrintParse(&x); jsonDebugPrintNode(pResult); jsonReturnJson(&x, pResult, ctx); }else{ @@ -2652,23 +2683,26 @@ static void jsonReplaceNode( } if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){ int k = jsonParseAddNode(p, JSON_STRING, n, z); + char *zCopy = sqlite3DbStrDup(0, z); if( k>0 ) p->aNode[k].jnFlags |= JNODE_RAW; - jsonParseAddCleanup(p, sqlite3_free, sqlite3DbStrDup(0,z)); - }else{ - int k= jsonParseAddNode(p, JSON_PATCH, 0, 0); - if( k>0 ){ - JsonParse *pPatch = jsonParseCached(pCtx, &pValue, pCtx); - if( pPatch==0 ){ - p->oom = 1; - break; - } - assert( pPatch->nJPRef>=1 ); - pPatch->nJPRef++; - p->aNode[k].jnFlags |= JNODE_PATCH; - p->aNode[k].eU = 5; - p->aNode[k].u.pPatch = pPatch->aNode; - jsonParseAddCleanup(p, (void(*)(void*))jsonParseFree, pPatch); + if( zCopy ){ + jsonParseAddCleanup(p, sqlite3_free, zCopy); + }else{ + sqlite3_result_error_nomem(pCtx); } + }else{ + JsonParse *pPatch = jsonParseCached(pCtx, &pValue, pCtx); + if( pPatch==0 ){ + p->oom = 1; + break; + } + jsonParseAddNodeArray(p, pPatch->aNode, pPatch->nNode); + /* The nodes copied out of pPatch and into p likely contain + ** u.zJContent pointers into pPatch->zJson. So preserve the + ** content of pPatch until p is destroyed. */ + assert( pPatch->nJPRef>=1 ); + pPatch->nJPRef++; + jsonParseAddCleanup(p, (void(*)(void*))jsonParseFree, pPatch); } break; } @@ -2703,7 +2737,6 @@ static void jsonReplaceFunc( pNode = jsonLookup(&x, zPath, 0, ctx); if( x.nErr ) goto replace_err; if( pNode ){ - pNode->jnFlags |= (u8)JNODE_REPLACE; jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]); } } @@ -2755,7 +2788,6 @@ static void jsonSetFunc( }else if( x.nErr ){ goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ - pNode->jnFlags |= (u8)JNODE_REPLACE; jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]); } } From 3d2a559ecb5087f554c36344ca790fb8f4502f2e Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 24 Jul 2023 22:45:59 +0000 Subject: [PATCH 05/16] Clean up some #defines in json. FossilOrigin-Name: 327fff501e36f75d4901c520123c5ca45e0e0da1d9cc8fa8fa877ceb68c686d2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 15 +++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 6a75e96be4..7f3392f540 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rework\sthe\sJSON\sparse\sstructure\sto\sfacilitate\sbetter\scaching.\s\sPasses\sall\ntests. -D 2023-07-24T22:34:26.093 +C Clean\sup\ssome\s#defines\sin\sjson. +D 2023-07-24T22:45:59.601 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e743eb83ad6d581bdfd355aee1d0f2c3025e955ec1799706ec4542b46e3af7f2 +F src/json.c a12c5517137b9ab77183b4a15084da264c32190f834fd3d45fda1eb77d9e327a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f2c063884685a79d5a787590447c292f51e898a98c9508159c788f505227ba85 -R fee4083d53f0bb74692c3aad54ce5646 +P ecdcb1ded76e9a0591bf7a2009679f49fc3aa639d3cc12406c6d29243ed8e1c5 +R c1e40d83b9a43b4c127658e4fcf06fd6 U drh -Z 5a89ce2654bac729f780024f319554e3 +Z 6091135e309dfa309be3fb90139aabae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0cc1f0a629..91a2b6e391 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecdcb1ded76e9a0591bf7a2009679f49fc3aa639d3cc12406c6d29243ed8e1c5 \ No newline at end of file +327fff501e36f75d4901c520123c5ca45e0e0da1d9cc8fa8fa877ceb68c686d2 \ No newline at end of file diff --git a/src/json.c b/src/json.c index b109a8759d..217c1070c9 100644 --- a/src/json.c +++ b/src/json.c @@ -109,14 +109,13 @@ static const char * const jsonType[] = { /* Bit values for the JsonNode.jnFlag field */ -#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ -#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ -#define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Target of a JSON_SUBST node */ - /* 0x10 Available for reuse */ -#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x40 /* Is a label of an object */ -#define JNODE_JSON5 0x80 /* Node contains JSON5 enhancements */ +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ +#define JNODE_REMOVE 0x04 /* Do not output */ +#define JNODE_REPLACE 0x08 /* Target of a JSON_SUBST node */ +#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x20 /* Is a label of an object */ +#define JNODE_JSON5 0x40 /* Node contains JSON5 enhancements */ /* A single node of parsed JSON From b715fe9d80b3656fd0afb82847932e6592306609 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 24 Jul 2023 23:27:05 +0000 Subject: [PATCH 06/16] It is an error to try to insert a BLOB value into JSON. FossilOrigin-Name: 28c6e964b1e2257527df02b352f02e135f7c4b764b8b41eda6461f9538f5d042 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 15 +++++++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7f3392f540..192ed6f6b3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\ssome\s#defines\sin\sjson. -D 2023-07-24T22:45:59.601 +C It\sis\san\serror\sto\stry\sto\sinsert\sa\sBLOB\svalue\sinto\sJSON. +D 2023-07-24T23:27:05.772 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a12c5517137b9ab77183b4a15084da264c32190f834fd3d45fda1eb77d9e327a +F src/json.c fc5b67025d4ca026d5c0696cfd5c265c1cd0fa2f669fbe78d268e955f853e75c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ecdcb1ded76e9a0591bf7a2009679f49fc3aa639d3cc12406c6d29243ed8e1c5 -R c1e40d83b9a43b4c127658e4fcf06fd6 +P 327fff501e36f75d4901c520123c5ca45e0e0da1d9cc8fa8fa877ceb68c686d2 +R bdfb959c0be321608fe703b9663eda08 U drh -Z 6091135e309dfa309be3fb90139aabae +Z 1ec3b6ff4cb1459575429b1da04d690b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 91a2b6e391..9eac82e0e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -327fff501e36f75d4901c520123c5ca45e0e0da1d9cc8fa8fa877ceb68c686d2 \ No newline at end of file +28c6e964b1e2257527df02b352f02e135f7c4b764b8b41eda6461f9538f5d042 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 217c1070c9..0ce68f1fcb 100644 --- a/src/json.c +++ b/src/json.c @@ -733,10 +733,12 @@ static void jsonReturnJson( sqlite3_result_error_nomem(pCtx); return; } - jsonInit(&s, pCtx); - jsonRenderNode(pParse, pNode, &s); - jsonResult(&s); - sqlite3_result_subtype(pCtx, JSON_SUBTYPE); + if( pParse->nErr==0 ){ + jsonInit(&s, pCtx); + jsonRenderNode(pParse, pNode, &s); + jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); + } } /* @@ -2705,6 +2707,11 @@ static void jsonReplaceNode( } break; } + case SQLITE_BLOB: { + sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); + p->nErr++; + break; + } } } From f02cc9a3248b07095847f9a5e93e092e4fa6e116 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 25 Jul 2023 15:08:18 +0000 Subject: [PATCH 07/16] Create the new RCStr class of strings and try to use them for JSON storage. FossilOrigin-Name: c1b8725089bb3d006ec69add28f4fcb3f4e79412c7f438b5b1067c2227e77b9c --- manifest | 21 ++++--- manifest.uuid | 2 +- src/json.c | 42 +++++++++----- src/printf.c | 143 ++++++++++++++++++++++++++++++++++++++++++++++++ src/sqliteInt.h | 52 ++++++++++++++++++ src/vdbemem.c | 23 ++++++++ 6 files changed, 258 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 192ed6f6b3..b4a115361a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C It\sis\san\serror\sto\stry\sto\sinsert\sa\sBLOB\svalue\sinto\sJSON. -D 2023-07-24T23:27:05.772 +C Create\sthe\snew\sRCStr\sclass\sof\sstrings\sand\stry\sto\suse\sthem\sfor\sJSON\sstorage. +D 2023-07-25T15:08:18.543 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c fc5b67025d4ca026d5c0696cfd5c265c1cd0fa2f669fbe78d268e955f853e75c +F src/json.c 3add12eb29f09a99dc260bcde427ea925f600cf0bb0d9abb0618a03d7e50eead F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -634,7 +634,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1 -F src/printf.c 84b7b4b647f336934a5ab2e7f0c52555833cc0778d2d60e016cca52ee8c6cd8f +F src/printf.c 21e410b0a3904dddb39ef1b245cfb991302022a6f0fc16f0a8a13539d6d2f713 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 @@ -643,7 +643,7 @@ F src/shell.c.in d320d8a13636de06d777cc1eab981caca304e175464e98183cf4ea68d93db81 F src/sqlite.h.in f999ef3642f381d69679b2516b430dbcb6c5a2a951b7f5e43dc4751b474a5774 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h dcb1a885e8b6cb78df618944b89d44361a99d0fe33e1bba2c150a855f7dc5599 +F src/sqliteInt.h 59b755dec944aa3b068e962ef53264de6fde3d6b6df2d5869ea3afdb7facdf60 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -714,7 +714,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c 40afb83ed848e235848ffdd3ba25adca4ba602111b8ed3b05ae3b1b12e0eacee +F src/vdbemem.c aee9ac636666616494d9a395d29efc3fe9e1404a9f043db81c82560b43b78f35 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2044,8 +2044,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 327fff501e36f75d4901c520123c5ca45e0e0da1d9cc8fa8fa877ceb68c686d2 -R bdfb959c0be321608fe703b9663eda08 +P 28c6e964b1e2257527df02b352f02e135f7c4b764b8b41eda6461f9538f5d042 +R 4c5edeff0b59e42b46536f2b483d14f6 +T *branch * json-opt-rcstr +T *sym-json-opt-rcstr * +T -sym-json-opt * U drh -Z 1ec3b6ff4cb1459575429b1da04d690b +Z 3f6f4e5856b228a13fb14e603baeb559 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9eac82e0e2..59c5808d17 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28c6e964b1e2257527df02b352f02e135f7c4b764b8b41eda6461f9538f5d042 \ No newline at end of file +c1b8725089bb3d006ec69add28f4fcb3f4e79412c7f438b5b1067c2227e77b9c \ No newline at end of file diff --git a/src/json.c b/src/json.c index 0ce68f1fcb..3bfa0c7484 100644 --- a/src/json.c +++ b/src/json.c @@ -191,7 +191,7 @@ static void jsonInit(JsonString *p, sqlite3_context *pCtx){ ** initial state. */ static void jsonReset(JsonString *p){ - if( !p->bStatic ) sqlite3_free(p->zBuf); + if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf); jsonZero(p); } @@ -212,7 +212,7 @@ static int jsonGrow(JsonString *p, u32 N){ char *zNew; if( p->bStatic ){ if( p->bErr ) return 1; - zNew = sqlite3_malloc64(nTotal); + zNew = sqlite3RCStrNew(nTotal); if( zNew==0 ){ jsonOom(p); return SQLITE_NOMEM; @@ -221,12 +221,12 @@ static int jsonGrow(JsonString *p, u32 N){ p->zBuf = zNew; p->bStatic = 0; }else{ - zNew = sqlite3_realloc64(p->zBuf, nTotal); - if( zNew==0 ){ - jsonOom(p); + p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); + if( p->zBuf==0 ){ + p->bErr = 1; + jsonZero(p); return SQLITE_NOMEM; } - p->zBuf = zNew; } p->nAlloc = nTotal; return SQLITE_OK; @@ -514,16 +514,26 @@ static void jsonAppendValue( /* Make the JSON in p the result of the SQL function. +** +** The JSON string is reset. */ static void jsonResult(JsonString *p){ if( p->bErr==0 ){ - jsonAppendChar(p, 0); - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed-1, - p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, - SQLITE_UTF8); - jsonZero(p); + if( p->bStatic ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, + SQLITE_UTF8); + }else{ + jsonAppendChar(p, 0); + p->nUsed--; + sqlite3RCStrRef(p->zBuf); + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + (void(*)(void*))sqlite3RCStrUnref, + SQLITE_UTF8); + } + }else{ + sqlite3_result_error_nomem(p->pCtx); } - assert( p->bStatic ); + jsonReset(p); } /************************************************************************** @@ -1885,7 +1895,7 @@ static JsonNode *jsonLookupStep( JsonNode *pRoot = &pParse->aNode[iRoot]; while( (pRoot->jnFlags & JNODE_REPLACE)!=0 ){ u32 idx = (u32)(pRoot - pParse->aNode); - u32 i = pParse->iSubst; + i = pParse->iSubst; while( 1 /*exit-by-break*/ ){ assert( inNode ); assert( pParse->aNode[i].eType==JSON_SUBST ); @@ -2950,7 +2960,8 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ assert( pStr->bStatic ); }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic ? SQLITE_TRANSIENT : + (void(*)(void*))sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); @@ -3058,7 +3069,8 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ assert( pStr->bStatic ); }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic ? SQLITE_TRANSIENT : + (void(*)(void*))sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); diff --git a/src/printf.c b/src/printf.c index 3fb1a322a0..7b84288168 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1366,3 +1366,146 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){ sqlite3_str_vappendf(p, zFormat, ap); va_end(ap); } + + +/***************************************************************************** +** Reference counted string storage +*****************************************************************************/ + +/* +** Increase the reference count of the string by one. +** +** The input parameter is returned. +*/ +char *sqlite3RCStrRef(char *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + p->nRCRef++; + return z; +} + +/* +** Decrease the reference count by one. Free the string when the +** reference count reaches zero. +*/ +void sqlite3RCStrUnref(char *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + assert( p->nRCRef>0 ); + assert( p->uMagic==SQLITE_RCSTR_MAGIC ); + if( p->nRCRef>=2 ){ + p->nRCRef--; + }else{ + if( p->xFree ) p->xFree(p->pAttach); +#ifdef SQLITE_DEBUG + p->uMagic = 0; +#endif + sqlite3_free(p); + } +} + +/* +** Return true if the reference count on the string is exactly one, meaning +** that the string can be modified. Return false if the reference count +** is greater than one. +*/ +int sqlite3RCStrIsWriteable(char *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + assert( p->nRCRef>0 ); + assert( p->uMagic==SQLITE_RCSTR_MAGIC ); + return p->nRCRef==1; +} + +/* +** Create a new string that is capable of holding N bytes of text, not counting +** the zero byte at the end. The string is uninitialized. +** +** The reference count is initially 1. Call sqlite3RCStrUnref() to free the +** newly allocated string. +** +** This routine returns 0 on an OOM. +*/ +char *sqlite3RCStrNew(u64 N){ + RCStr *p = sqlite3_malloc64( N + sizeof(*p) ); + if( p==0 ) return 0; + p->nRCRef = 1; + p->xFree = 0; + p->pAttach = 0; +#ifdef SQLITE_DEBUG + p->uMagic = SQLITE_RCSTR_MAGIC; +#endif + return (char*)&p[1]; +} + +/* +** Return the number of bytes allocated to the string. The value returned +** does not include the space for the zero-terminator at the end. +*/ +u64 sqlite3RCStrSize(char *z){ + RCStr *p = (RCStr*)z; + u64 N; + assert( p!=0 ); + p--; + assert( p->nRCRef>0 ); + assert( p->uMagic==SQLITE_RCSTR_MAGIC ); + N = sqlite3_msize(p); + N -= sizeof(p) + 1; + return N; +} + +/* +** Change the size of the string so that it is able to hold N bytes. +** The string might be reallocated, so return the new allocation. +*/ +char *sqlite3RCStrResize(char *z, u64 N){ + RCStr *p = (RCStr*)z; + RCStr *pNew; + assert( p!=0 ); + p--; + assert( p->nRCRef==1 ); + assert( p->uMagic==SQLITE_RCSTR_MAGIC ); + pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1); + if( pNew==0 ){ + sqlite3_free(p); + return 0; + }else{ + return (char*)&pNew[1]; + } +} + +/* +** Add a new attachment to the string. +** +** A string may have no more than one attachment. When a new attachment +** is added, any prior attachment is destroyed. Remove an attachment +** by adding a zero-attachment. +*/ +void sqlite3RCStrAttach(char *z, void *pAttach, void(*xFree)(void*)){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + assert( p->nRCRef>0 ); + assert( p->uMagic==SQLITE_RCSTR_MAGIC ); + if( p->xFree ) p->xFree(p->pAttach); + p->xFree = xFree; + p->pAttach = pAttach; +} + +/* +** Return the attachment associated with a string if the attachment +** has the destructure specified in the second argument. If the +** string has no attachment or if the destructor does not match, +** then return a NULL pointr. +*/ +void *sqlite3RCStrGetAttachment(char *z, void(*xFree)(void*)){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + assert( p->nRCRef>0 ); + assert( p->uMagic==SQLITE_RCSTR_MAGIC ); + return p->xFree==xFree ? p->pAttach : 0; +} diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f214862f74..adf4d34dec 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1281,6 +1281,7 @@ typedef struct Parse Parse; typedef struct ParseCleanup ParseCleanup; typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; +typedef struct RCStr RCStr; typedef struct RenameToken RenameToken; typedef struct Returning Returning; typedef struct RowSet RowSet; @@ -4061,6 +4062,47 @@ struct sqlite3_str { #define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) +/* +** The following object is the header for an "RCStr" or "reference-counted +** string". An RCStr is passed around and used like any other char* +** that has been dynamically allocated. The important interface +** difference is that it uses sqlite3RCStrUnref() as its destructor +** rather than sqlite3_free(). Other than that the two are interchangeable. +** +** Thus to return an RCStr object as the result of an SQL function use: +** +** sqlite3_result_text64(ctx,z,sz,sqlite3RCStrUnref,SQLITE_UTF8) +** ^^^^^^^^^^^^^^^^^ +** Instead of sqlite3_free() or similar +** +** An SQL function can check its arguments to see if they are RCStr +** strings using the sqlite3ValueIsOfClass() function: +** +** sqlite3ValueIsOfClass(argv[i], sqlite3RCStrUnref); +** +** An RCStr string might be better than an ordinary string in some cases +** because: +** +** (1) You can duplicate it using sqlite3RCStrRef(x). +** +** (2) You can also add an associated object to the string. For +** example, if the string is JSON, perhaps the associated object +** is a parse of that JSON. +** +** Methods for an RCStr string begin with "sqlite3RCStr...". +*/ +struct RCStr { + u32 nRCRef; /* Number of references */ +#ifdef SQLITE_DEBUG + u32 uMagic; /* Magic number for sanity checking */ +#endif + void *pAttach; /* Attachment to this string */ + void (*xFree)(void*); /* Destructor for the attachment */ +}; + +/* The Magic number used by RCStr for sanity checking. SQLITE_DEBUG only. */ +#define SQLITE_RCSTR_MAGIC 0x3dc05d54 + /* ** A pointer to this structure is used to communicate information @@ -5180,6 +5222,7 @@ void sqlite3FileSuffix3(const char*, char*); u8 sqlite3GetBoolean(const char *z,u8); const void *sqlite3ValueText(sqlite3_value*, u8); +int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*)); int sqlite3ValueBytes(sqlite3_value*, u8); void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); @@ -5287,6 +5330,15 @@ void sqlite3OomClear(sqlite3*); int sqlite3ApiExit(sqlite3 *db, int); int sqlite3OpenTempDatabase(Parse *); +char *sqlite3RCStrRef(char*); +void sqlite3RCStrUnref(char*); +char *sqlite3RCStrNew(u64); +u64 sqlite3RCStrSize(char*); +char *sqlite3RCStrResize(char*,u64); +int sqlite3RCStrIsWriteable(char*); +void sqlite3RCStrAttach(char*, void*, void(*)(void*)); +void *sqlite3RCStrGetAttachment(char*,void(*)(void*)); + void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); int sqlite3StrAccumEnlarge(StrAccum*, i64); char *sqlite3StrAccumFinish(StrAccum*); diff --git a/src/vdbemem.c b/src/vdbemem.c index b5a794ae8f..87dfbbebd8 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -333,6 +333,11 @@ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ pMem->flags |= MEM_Term; return; } + if( pMem->xDel==(void(*)(void*))sqlite3RCStrUnref ){ + /* Blindly assume that all RCStr objects are zero-terminated */ + pMem->flags |= MEM_Term; + return; + } }else if( pMem->szMalloc>0 && pMem->szMalloc >= pMem->n+1 ){ pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; @@ -1363,6 +1368,24 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return valueToText(pVal, enc); } +/* Return true if sqlit3_value object pVal is a string or blob value +** that uses the destructor specified in the second argument. +** +** TODO: Maybe someday promote this interface into a published API so +** that third-party extensions can get access to it? +*/ +int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){ + if( ALWAYS(pVal!=0) + && (pVal->flags & (MEM_Str|MEM_Blob))!=0 + && (pVal->flags & MEM_Dyn)!=0 + && pVal->xDel==xFree + ){ + return 1; + }else{ + return 0; + } +} + /* ** Create a new sqlite3_value object. */ From 4bca1248455c6b8c4c79ac953461f4726429ed02 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 25 Jul 2023 15:43:01 +0000 Subject: [PATCH 08/16] Fix a minor problem with error reporting in JSON. FossilOrigin-Name: c456e4a8999066cd96246327101b3cca78294511a71a2ac07939bb702bfcb5f4 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b4a115361a..f3393df05d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Create\sthe\snew\sRCStr\sclass\sof\sstrings\sand\stry\sto\suse\sthem\sfor\sJSON\sstorage. -D 2023-07-25T15:08:18.543 +C Fix\sa\sminor\sproblem\swith\serror\sreporting\sin\sJSON. +D 2023-07-25T15:43:01.908 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 3add12eb29f09a99dc260bcde427ea925f600cf0bb0d9abb0618a03d7e50eead +F src/json.c 8f44f1aadfe1967af70dc9e6390823806f5a42cc301f11df435a1c3d4950308d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,11 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 28c6e964b1e2257527df02b352f02e135f7c4b764b8b41eda6461f9538f5d042 -R 4c5edeff0b59e42b46536f2b483d14f6 -T *branch * json-opt-rcstr -T *sym-json-opt-rcstr * -T -sym-json-opt * +P c1b8725089bb3d006ec69add28f4fcb3f4e79412c7f438b5b1067c2227e77b9c +R 625da2283c68c7b3c0e2eb949802be15 U drh -Z 3f6f4e5856b228a13fb14e603baeb559 +Z 238deba72513ff58266287df7bec917f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 59c5808d17..70df72c921 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1b8725089bb3d006ec69add28f4fcb3f4e79412c7f438b5b1067c2227e77b9c \ No newline at end of file +c456e4a8999066cd96246327101b3cca78294511a71a2ac07939bb702bfcb5f4 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3bfa0c7484..18c4721455 100644 --- a/src/json.c +++ b/src/json.c @@ -530,7 +530,7 @@ static void jsonResult(JsonString *p){ (void(*)(void*))sqlite3RCStrUnref, SQLITE_UTF8); } - }else{ + }else if( p->bErr==1 ){ sqlite3_result_error_nomem(p->pCtx); } jsonReset(p); From 440e696d5e0b31a2542a01b3fba9eb356fea52ad Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 25 Jul 2023 18:28:03 +0000 Subject: [PATCH 09/16] Incremental improvements to JSON parsing - trying to fold in the RCStr object. FossilOrigin-Name: 4cb15d934a85ebc290fe6dd8cd3bd47b159561ca75d72bbffef30b9ea4623b09 --- manifest | 12 ++-- manifest.uuid | 2 +- src/json.c | 184 +++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 166 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index f3393df05d..09e5a7cfff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sminor\sproblem\swith\serror\sreporting\sin\sJSON. -D 2023-07-25T15:43:01.908 +C Incremental\simprovements\sto\sJSON\sparsing\s-\strying\sto\sfold\sin\sthe\sRCStr\sobject. +D 2023-07-25T18:28:03.341 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 8f44f1aadfe1967af70dc9e6390823806f5a42cc301f11df435a1c3d4950308d +F src/json.c 28fe8ed9e63293fd5708d129d4edd4a634887aabb0364685195974e35ee10762 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c1b8725089bb3d006ec69add28f4fcb3f4e79412c7f438b5b1067c2227e77b9c -R 625da2283c68c7b3c0e2eb949802be15 +P c456e4a8999066cd96246327101b3cca78294511a71a2ac07939bb702bfcb5f4 +R 2b5df60a4958d2a70ce11e3569ae2bab U drh -Z 238deba72513ff58266287df7bec917f +Z d10e127820cbbcdc4a84e1b86eb42adb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 70df72c921..a811037656 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c456e4a8999066cd96246327101b3cca78294511a71a2ac07939bb702bfcb5f4 \ No newline at end of file +4cb15d934a85ebc290fe6dd8cd3bd47b159561ca75d72bbffef30b9ea4623b09 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 18c4721455..194d0421eb 100644 --- a/src/json.c +++ b/src/json.c @@ -142,7 +142,7 @@ struct JsonParse { u32 nNode; /* Number of slots of aNode[] used */ u32 nAlloc; /* Number of slots of aNode[] allocated */ JsonNode *aNode; /* Array of nodes containing the parse */ - const char *zJson; /* Original JSON string */ + char *zJson; /* Original JSON string */ u32 *aUp; /* Index of parent of each node */ JsonTask *pClean; /* Cleanup operations prior to freeing this object */ u16 iDepth; /* Nesting depth */ @@ -150,6 +150,8 @@ struct JsonParse { u8 oom; /* Set to true if out of memory */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 nJPRef; /* Number of references to this object */ + u8 bOwnsJson; /* This object owns zJson and response for freeing it */ + u8 isMod; /* aNode contains edits from the original zJson */ int nJson; /* Length of the zJson string in bytes */ u32 iErr; /* Error location in zJson[] */ u32 iSubst; /* Last known JSON_SUBST node */ @@ -565,24 +567,41 @@ static void jsonParseReset(JsonParse *pParse){ sqlite3_free(pTask); } assert( pParse->nJPRef<=1 ); - sqlite3_free(pParse->aNode); - pParse->aNode = 0; + if( pParse->aNode ){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + } pParse->nNode = 0; pParse->nAlloc = 0; - sqlite3_free(pParse->aUp); - pParse->aUp = 0; + if( pParse->aUp ){ + sqlite3_free(pParse->aUp); + pParse->aUp = 0; + } + if( pParse->bOwnsJson ){ + /* Order operations so that if the destructor for pParse->zJson + ** invokes jsonParseFree(), the recursion will terminate harmlessly */ + char *z = pParse->zJson; + pParse->zJson = 0; + pParse->bOwnsJson = 0; + sqlite3RCStrUnref(z); + } } /* ** Free a JsonParse object that was obtained from sqlite3_malloc(). +** +** Note that destroying JsonParse might call sqlite3RCStrUnref() to +** destroy the zJson value. The RCStr object might recursively invoke +** JsonParse to destroy this pParse object again. Take care to ensure +** that this recursive destructor sequence terminates harmlessly. */ static void jsonParseFree(JsonParse *pParse){ if( pParse->nJPRef>1 ){ pParse->nJPRef--; - return; + }else{ + jsonParseReset(pParse); + sqlite3_free(pParse); } - jsonParseReset(pParse); - sqlite3_free(pParse); } /* @@ -1218,7 +1237,7 @@ static const struct NanInfName { ** ** Special return values: ** -** 0 End if input +** 0 End of input ** -1 Syntax error ** -2 '}' seen ** -3 ']' seen @@ -1685,7 +1704,7 @@ json_parse_restart: static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ sqlite3_context *pCtx, /* Report errors here */ - const char *zJson /* Input JSON text to be parsed */ + char *zJson /* Input JSON text to be parsed */ ){ int i; memset(pParse, 0, sizeof(*pParse)); @@ -1719,6 +1738,121 @@ static int jsonParse( return 0; } +#if 0 +/* +** This is a destructor for JSON strings. We make it a separate function +** so that the sqlite3ValueIsOfClass() function can be used to unambiguously +** identify sqlite3_value objects that are known JSON strings. +*/ +static void jsonClass(void *p){ + sqlite3RCStrUnref((char*)p); +} +#endif + +#if 0 +/* +** Process SQL function argument pJson as a JSON string. Return a pointer +** to its parse. +** +** If any error is encountered, return a NULL pointer and leave an error +** message in pCtx. +*/ +static JsonParse *jsonParseFromFunctionArg( + sqlite3_value *pJson, /* An SQL function argument containing JSON */ + sqlite3_context *pCtx, /* For accessing the cache */ + sqlite3_context *pErrCtx, /* For reporting errors */ + int bUnchng /* Only accept cached parse that are unchanged */ +){ + JsonParse *pParse; + char *zJson; + int nJson; + JsonParse *pMatch = 0; + int iKey; + int iMinKey = 0; + u32 iMinHold = 0xffffffff; + u32 iMaxHold = 0; + + char *zJson = (char*)sqlite3_value_text(pJson); + int nJson = sqlite3_value_bytes(pJson); + if( zJson==0 ) goto json_parse_value_nomem; + if( sqlite3ValueIsOfClass(pJson, jsonClass) ){ + assert( zJson!=0 ); + (void)sqlite3RCStrRef(zJson); + pParse = sqlite3RCStrGetAttachment(zJson, (void(*)(void*)jsonParseFree); + if( pParse ){ + return pParse; + } + }else{ + char *z = zJson; + zJson = sqlite3RCStrNew( nJson ); + if( zJson==0 ) goto json_parse_value_nomem; + assert( strlen(z)==nJson ); + memcpy(zJson, z, nJson+1); + } + + /* At this point zJson is an RCStr object that does not yet have a + ** parse. This procedure is holding its own reference to that zJson + ** and needs to release it, or hand it off, prior to returning. + ** + ** The next step is to try to find a parse of the JSON that already + ** exists in cache. + */ + for(iKey=0; iKeynJson==nJson + && (p->isMod==0 || bUnchng==0) + && memcmp(p->zJson,zJson,nJson)==0 + ){ + p->nErr = 0; + pMatch = p; + }else if( p->iHoldiHold; + iMinKey = iKey; + } + if( p->iHold>iMaxHold ){ + iMaxHold = p->iHold; + } + } + + if( pMatch ){ + pParse = pMatch; + pParse->iHold = iMaxHold+1; + }else{ + /* No parse of zJson could be found in cache. So parse it afresh. + */ + pParse = sqlite3_malloc64( sizeof(JsonParse) ); + if( pParse==0 ) goto json_parse_value_nomem; + if( jsonParse(pParse, pCtx, zJson) ){ + sqlite3_free(pParse); + sqlite3RCStrUnref(zJson); + return 0; + } + pParse->bOwnsJson = 1; + pParse->nJson = nJson; + pParse->nJPRef = 1; + pParse->iHold = iMaxHold+1; + sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, pParse, + (void(*)(void*))jsonParseFree); + pParse = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); + } + if( pParse ){ + pParse->nJPRef++; /* The caller will own this object */ + } + return pParse; + +json_parse_value_nomem: + if( zJson ) sqlite3RCStrUnref(zJson); + if( pErrCtx ) sqlite3_result_error_nomem(pCtx); + return 0; +} +#endif + /* Mark node i of pParse as being a child of iParent. Call recursively ** to fill in all the descendants of node i. */ @@ -1786,11 +1920,11 @@ static int jsonParseFindParents(JsonParse *pParse){ */ static JsonParse *jsonParseCached( sqlite3_context *pCtx, - sqlite3_value **argv, + sqlite3_value *pJson, sqlite3_context *pErrCtx ){ - const char *zJson = (const char*)sqlite3_value_text(argv[0]); - int nJson = sqlite3_value_bytes(argv[0]); + char *zJson = (char*)sqlite3_value_text(pJson); + int nJson = sqlite3_value_bytes(pJson); JsonParse *p; JsonParse *pMatch = 0; int iKey; @@ -2223,7 +2357,7 @@ static void jsonParseFunc( u32 i; assert( argc==1 ); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return; jsonParseFindParents(&x); jsonInit(&s, ctx); for(i=0; inNode ); if( argc==2 ){ @@ -2393,7 +2527,7 @@ static void jsonExtractFunc( JsonString jx; if( argc<2 ) return; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseCached(ctx, argv[0], ctx); if( p==0 ) return; if( argc==2 ){ /* With a single PATH argument */ @@ -2547,8 +2681,8 @@ static void jsonPatchFunc( JsonNode *pResult; /* The result of the merge */ UNUSED_PARAMETER(argc); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ + if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&y, ctx, (char*)sqlite3_value_text(argv[1])) ){ jsonParseReset(&x); return; } @@ -2624,7 +2758,7 @@ static void jsonRemoveFunc( u32 i; if( argc<1 ) return; - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return; assert( x.nNode ); for(i=1; i<(u32)argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); @@ -2702,7 +2836,7 @@ static void jsonReplaceNode( sqlite3_result_error_nomem(pCtx); } }else{ - JsonParse *pPatch = jsonParseCached(pCtx, &pValue, pCtx); + JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx); if( pPatch==0 ){ p->oom = 1; break; @@ -2746,7 +2880,7 @@ static void jsonReplaceFunc( jsonWrongNumArgs(ctx, "replace"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return; assert( x.nNode ); for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); @@ -2792,7 +2926,7 @@ static void jsonSetFunc( jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return; assert( x.nNode ); for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); @@ -2830,7 +2964,7 @@ static void jsonTypeFunc( const char *zPath; JsonNode *pNode; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseCached(ctx, argv[0], ctx); if( p==0 ) return; if( argc==2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); @@ -2857,7 +2991,7 @@ static void jsonValidFunc( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv, 0); + p = jsonParseCached(ctx, argv[0], 0); if( p==0 || p->oom ){ sqlite3_result_error_nomem(ctx); sqlite3_free(p); @@ -2903,7 +3037,7 @@ static void jsonErrorFunc( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv, 0); + p = jsonParseCached(ctx, argv[0], 0); if( p==0 || p->oom ){ sqlite3_result_error_nomem(ctx); sqlite3_free(p); From 59b8e666f669a97e7d62fd1a6e24daed446f027e Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 25 Jul 2023 20:26:47 +0000 Subject: [PATCH 10/16] Clarify ownership of the various objects involved in parsing JSON. FossilOrigin-Name: afe02a398a16d51bd7482b6fbe2fbd15d9ac4fd9cdbc9d2bf81f38b3391fc567 --- manifest | 14 ++-- manifest.uuid | 2 +- src/json.c | 205 ++++++++++++++++---------------------------------- src/printf.c | 2 +- 4 files changed, 75 insertions(+), 148 deletions(-) diff --git a/manifest b/manifest index 09e5a7cfff..437038d380 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Incremental\simprovements\sto\sJSON\sparsing\s-\strying\sto\sfold\sin\sthe\sRCStr\sobject. -D 2023-07-25T18:28:03.341 +C Clarify\sownership\sof\sthe\svarious\sobjects\sinvolved\sin\sparsing\sJSON. +D 2023-07-25T20:26:47.054 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 28fe8ed9e63293fd5708d129d4edd4a634887aabb0364685195974e35ee10762 +F src/json.c 97f8c20c1cfefbc54e2b8b2143a24aae4993c12de6e74764af443e033b01b013 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -634,7 +634,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1 -F src/printf.c 21e410b0a3904dddb39ef1b245cfb991302022a6f0fc16f0a8a13539d6d2f713 +F src/printf.c 1406ade1451adfa4374d8e9bbb8606109742c1216f5dbc95e011bf721fd91365 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c456e4a8999066cd96246327101b3cca78294511a71a2ac07939bb702bfcb5f4 -R 2b5df60a4958d2a70ce11e3569ae2bab +P 4cb15d934a85ebc290fe6dd8cd3bd47b159561ca75d72bbffef30b9ea4623b09 +R cbb913300d23d0635674a5fded1ff5ab U drh -Z d10e127820cbbcdc4a84e1b86eb42adb +Z f943de02bc9e737c29b8eb0aa52b28c4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a811037656..f222cbf323 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4cb15d934a85ebc290fe6dd8cd3bd47b159561ca75d72bbffef30b9ea4623b09 \ No newline at end of file +afe02a398a16d51bd7482b6fbe2fbd15d9ac4fd9cdbc9d2bf81f38b3391fc567 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 194d0421eb..6cb429f407 100644 --- a/src/json.c +++ b/src/json.c @@ -171,6 +171,17 @@ struct JsonParse { ** Utility routines for dealing with JsonString objects **************************************************************************/ +#if 0 +/* +** This is a destructor for JSON strings. We make it a separate function +** so that the sqlite3ValueIsOfClass() function can be used to unambiguously +** identify sqlite3_value objects that are known JSON strings. +*/ +static void jsonStringClass(void *p){ + sqlite3RCStrUnref((char*)p); +} +#endif + /* Set the JsonString object to an empty string */ static void jsonZero(JsonString *p){ @@ -1700,16 +1711,27 @@ json_parse_restart: ** pParse. ** ** pParse is uninitialized when this routine is called. +** +** pParse->nJPRef set to 1. The caller becomes the owner of the +** the JsonParse object. +** +** pParse->bOwnsJson is to bTakeJson. If bTakeJson is 1, the newly initialized +** JsonParse object will become the own the zJson input string. If bTakeJson +** is 0, then the caller is responsible for preserving zJson for the lifetime +** of the JsonParse object. */ static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ sqlite3_context *pCtx, /* Report errors here */ - char *zJson /* Input JSON text to be parsed */ + char *zJson, /* Input JSON text to be parsed */ + int bTakeJson /* Assume ownership of zJson if true */ ){ int i; memset(pParse, 0, sizeof(*pParse)); if( zJson==0 ) return 1; pParse->zJson = zJson; + pParse->bOwnsJson = bTakeJson; + pParse->nJPRef = 1; i = jsonParseValue(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ @@ -1738,120 +1760,6 @@ static int jsonParse( return 0; } -#if 0 -/* -** This is a destructor for JSON strings. We make it a separate function -** so that the sqlite3ValueIsOfClass() function can be used to unambiguously -** identify sqlite3_value objects that are known JSON strings. -*/ -static void jsonClass(void *p){ - sqlite3RCStrUnref((char*)p); -} -#endif - -#if 0 -/* -** Process SQL function argument pJson as a JSON string. Return a pointer -** to its parse. -** -** If any error is encountered, return a NULL pointer and leave an error -** message in pCtx. -*/ -static JsonParse *jsonParseFromFunctionArg( - sqlite3_value *pJson, /* An SQL function argument containing JSON */ - sqlite3_context *pCtx, /* For accessing the cache */ - sqlite3_context *pErrCtx, /* For reporting errors */ - int bUnchng /* Only accept cached parse that are unchanged */ -){ - JsonParse *pParse; - char *zJson; - int nJson; - JsonParse *pMatch = 0; - int iKey; - int iMinKey = 0; - u32 iMinHold = 0xffffffff; - u32 iMaxHold = 0; - - char *zJson = (char*)sqlite3_value_text(pJson); - int nJson = sqlite3_value_bytes(pJson); - if( zJson==0 ) goto json_parse_value_nomem; - if( sqlite3ValueIsOfClass(pJson, jsonClass) ){ - assert( zJson!=0 ); - (void)sqlite3RCStrRef(zJson); - pParse = sqlite3RCStrGetAttachment(zJson, (void(*)(void*)jsonParseFree); - if( pParse ){ - return pParse; - } - }else{ - char *z = zJson; - zJson = sqlite3RCStrNew( nJson ); - if( zJson==0 ) goto json_parse_value_nomem; - assert( strlen(z)==nJson ); - memcpy(zJson, z, nJson+1); - } - - /* At this point zJson is an RCStr object that does not yet have a - ** parse. This procedure is holding its own reference to that zJson - ** and needs to release it, or hand it off, prior to returning. - ** - ** The next step is to try to find a parse of the JSON that already - ** exists in cache. - */ - for(iKey=0; iKeynJson==nJson - && (p->isMod==0 || bUnchng==0) - && memcmp(p->zJson,zJson,nJson)==0 - ){ - p->nErr = 0; - pMatch = p; - }else if( p->iHoldiHold; - iMinKey = iKey; - } - if( p->iHold>iMaxHold ){ - iMaxHold = p->iHold; - } - } - - if( pMatch ){ - pParse = pMatch; - pParse->iHold = iMaxHold+1; - }else{ - /* No parse of zJson could be found in cache. So parse it afresh. - */ - pParse = sqlite3_malloc64( sizeof(JsonParse) ); - if( pParse==0 ) goto json_parse_value_nomem; - if( jsonParse(pParse, pCtx, zJson) ){ - sqlite3_free(pParse); - sqlite3RCStrUnref(zJson); - return 0; - } - pParse->bOwnsJson = 1; - pParse->nJson = nJson; - pParse->nJPRef = 1; - pParse->iHold = iMaxHold+1; - sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, pParse, - (void(*)(void*))jsonParseFree); - pParse = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); - } - if( pParse ){ - pParse->nJPRef++; /* The caller will own this object */ - } - return pParse; - -json_parse_value_nomem: - if( zJson ) sqlite3RCStrUnref(zJson); - if( pErrCtx ) sqlite3_result_error_nomem(pCtx); - return 0; -} -#endif /* Mark node i of pParse as being a child of iParent. Call recursively ** to fill in all the descendants of node i. @@ -1905,18 +1813,28 @@ static int jsonParseFindParents(JsonParse *pParse){ ** Obtain a complete parse of the JSON found in the first argument ** of the argv array. Use the sqlite3_get_auxdata() cache for this ** parse if it is available. If the cache is not available or if it -** is no longer valid, parse the JSON again and return the new parse, -** and also register the new parse so that it will be available for +** is no longer valid, parse the JSON again and return the new parse. +** Also register the new parse so that it will be available for ** future sqlite3_get_auxdata() calls. ** ** If an error occurs and pErrCtx!=0 then report the error on pErrCtx ** and return NULL. ** -** If an error occurs and pErrCtx==0 then return the Parse object with -** JsonParse.nErr non-zero. If the caller invokes this routine with -** pErrCtx==0 and it gets back a JsonParse with nErr!=0, then the caller -** is responsible for invoking jsonParseFree() on the returned value. -** But the caller may invoke jsonParseFree() *only* if pParse->nErr!=0. +** The returned pointer (if it is not NULL) is owned by the cache in +** most cases, not the caller. The caller does NOT need to invoke +** jsonParseFree(), in most cases. +** +** Except, if an error occurs and pErrCtx==0 then return the JsonParse +** object with JsonParse.nErr non-zero and the caller will own the JsonParse +** object. In that case, it will be the responsibility of the caller to +** invoke jsonParseFree(). To summarize: +** +** pErrCtx!=0 || p->nErr==0 ==> Return value p is owned by the +** cache. Call does not need to +** free it. +** +** pErrCtx==0 && p->nErr!=0 ==> Return value is owned by the caller +** and so the caller must free it. */ static JsonParse *jsonParseCached( sqlite3_context *pCtx, @@ -1931,6 +1849,7 @@ static JsonParse *jsonParseCached( int iMinKey = 0; u32 iMinHold = 0xffffffff; u32 iMaxHold = 0; + if( zJson==0 ) return 0; for(iKey=0; iKeynErr = 0; pMatch->iHold = iMaxHold+1; + assert( pMatch->nJPRef>0 ); /* pMatch is owned by the cache */ return pMatch; } - p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ){ sqlite3_result_error_nomem(pCtx); return 0; } memset(p, 0, sizeof(*p)); - p->zJson = (char*)&p[1]; - memcpy((char*)p->zJson, zJson, nJson+1); - if( jsonParse(p, pErrCtx, p->zJson) ){ - if( pErrCtx==0 ){ - p->nErr = 1; - return p; - } + p->zJson = sqlite3RCStrNew( nJson ); + if( p->zJson==0 ){ sqlite3_free(p); + sqlite3_result_error_nomem(pCtx); + return 0; + } + memcpy(p->zJson, zJson, nJson); + p->zJson[nJson] = 0; + if( jsonParse(p, pErrCtx, p->zJson, 1) ){ + if( pErrCtx==0 ){ + p->nErr = 1; + assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */ + return p; + } + jsonParseFree(p); return 0; } - p->nJPRef = 1; p->nJson = nJson; p->iHold = iMaxHold+1; + /* Transfer ownership of the new JsonParse to the cache */ sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, (void(*)(void*))jsonParseFree); return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); @@ -2357,7 +2284,7 @@ static void jsonParseFunc( u32 i; assert( argc==1 ); - if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return; jsonParseFindParents(&x); jsonInit(&s, ctx); for(i=0; izJson; + const char *z = (const char*)sqlite3_value_text(argv[0]); for(i=0; iiErr && ALWAYS(z[i]); i++){ if( (z[i]&0xc0)!=0x80 ) n++; } @@ -3639,7 +3566,7 @@ static int jsonEachFilter( p->zJson = sqlite3_malloc64( n+1 ); if( p->zJson==0 ) return SQLITE_NOMEM; memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson) ){ + if( jsonParse(&p->sParse, 0, p->zJson, 0) ){ int rc = SQLITE_NOMEM; if( p->sParse.oom==0 ){ sqlite3_free(cur->pVtab->zErrMsg); diff --git a/src/printf.c b/src/printf.c index 7b84288168..222e70be29 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1430,7 +1430,7 @@ int sqlite3RCStrIsWriteable(char *z){ ** This routine returns 0 on an OOM. */ char *sqlite3RCStrNew(u64 N){ - RCStr *p = sqlite3_malloc64( N + sizeof(*p) ); + RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 ); if( p==0 ) return 0; p->nRCRef = 1; p->xFree = 0; From 0f200bc580eda625f48866d0311ce6f57bfa531a Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Jul 2023 00:48:45 +0000 Subject: [PATCH 11/16] Add the JsonParse.zAlt field to old revised JSON text after a change. Demonstrate that this elminates the need for reparsing after a change by using it in the json_remove() function. This is an incremental check-in containing lots of cruft. FossilOrigin-Name: f930b139d6db0ee799bc90397b225175103c4bf22923d1c9cbcd32509adc1738 --- manifest | 18 +++--- manifest.uuid | 2 +- src/json.c | 146 +++++++++++++++++++++++++++++++++++------------- src/printf.c | 10 +++- src/sqliteInt.h | 10 ++-- src/vdbemem.c | 2 + 6 files changed, 134 insertions(+), 54 deletions(-) diff --git a/manifest b/manifest index 437038d380..875c317328 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarify\sownership\sof\sthe\svarious\sobjects\sinvolved\sin\sparsing\sJSON. -D 2023-07-25T20:26:47.054 +C Add\sthe\sJsonParse.zAlt\sfield\sto\sold\srevised\sJSON\stext\safter\sa\schange.\nDemonstrate\sthat\sthis\selminates\sthe\sneed\sfor\sreparsing\safter\sa\schange\nby\susing\sit\sin\sthe\sjson_remove()\sfunction.\s\sThis\sis\san\sincremental\scheck-in\ncontaining\slots\sof\scruft. +D 2023-07-26T00:48:45.768 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 97f8c20c1cfefbc54e2b8b2143a24aae4993c12de6e74764af443e033b01b013 +F src/json.c a7823b0593eac42a608f8439e6b1f28022ac5fce9aa81a82856e48dd07ab601d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -634,7 +634,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1 -F src/printf.c 1406ade1451adfa4374d8e9bbb8606109742c1216f5dbc95e011bf721fd91365 +F src/printf.c 56e362bbefbe61b5b211f1886e3cdb9b25d8ba817c12df3a15f11b9bdc152efd F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 @@ -643,7 +643,7 @@ F src/shell.c.in d320d8a13636de06d777cc1eab981caca304e175464e98183cf4ea68d93db81 F src/sqlite.h.in f999ef3642f381d69679b2516b430dbcb6c5a2a951b7f5e43dc4751b474a5774 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 59b755dec944aa3b068e962ef53264de6fde3d6b6df2d5869ea3afdb7facdf60 +F src/sqliteInt.h 29c8e6d0b6dfa9c2c356f23ab8349d7fac3679d5f7cfc01d4ee45c9171db2960 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -714,7 +714,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c aee9ac636666616494d9a395d29efc3fe9e1404a9f043db81c82560b43b78f35 +F src/vdbemem.c 46ea371b10c3573cf9287d385411fbcc6cd42099ad3505d20e33f2f3ed91537d F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4cb15d934a85ebc290fe6dd8cd3bd47b159561ca75d72bbffef30b9ea4623b09 -R cbb913300d23d0635674a5fded1ff5ab +P afe02a398a16d51bd7482b6fbe2fbd15d9ac4fd9cdbc9d2bf81f38b3391fc567 +R 33c717c250039ffbe53dc8d7b2094929 U drh -Z f943de02bc9e737c29b8eb0aa52b28c4 +Z 96ad60bdf5cbde0ec2b901493a583f6e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f222cbf323..dce1119850 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afe02a398a16d51bd7482b6fbe2fbd15d9ac4fd9cdbc9d2bf81f38b3391fc567 \ No newline at end of file +f930b139d6db0ee799bc90397b225175103c4bf22923d1c9cbcd32509adc1738 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6cb429f407..d7d464442b 100644 --- a/src/json.c +++ b/src/json.c @@ -142,7 +142,8 @@ struct JsonParse { u32 nNode; /* Number of slots of aNode[] used */ u32 nAlloc; /* Number of slots of aNode[] allocated */ JsonNode *aNode; /* Array of nodes containing the parse */ - char *zJson; /* Original JSON string */ + char *zJson; /* Original JSON string (before edits) */ + char *zAlt; /* Revised JSON after edits applied. Maybe NULL */ u32 *aUp; /* Index of parent of each node */ JsonTask *pClean; /* Cleanup operations prior to freeing this object */ u16 iDepth; /* Nesting depth */ @@ -151,8 +152,10 @@ struct JsonParse { u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 nJPRef; /* Number of references to this object */ u8 bOwnsJson; /* This object owns zJson and response for freeing it */ - u8 isMod; /* aNode contains edits from the original zJson */ + u8 useMod; /* Actually use the edits contain inside aNode */ + u8 hasMod; /* aNode contains edits from the original zJson */ int nJson; /* Length of the zJson string in bytes */ + int nAlt; /* Length of alternative JSON string zAlt, in bytes */ u32 iErr; /* Error location in zJson[] */ u32 iSubst; /* Last known JSON_SUBST node */ u32 iHold; /* Replace cache line with the lowest iHold value */ @@ -245,6 +248,16 @@ static int jsonGrow(JsonString *p, u32 N){ return SQLITE_OK; } +/* Try to force the string to be an RCStr string, rather than a +** static string. Return true on success. The only reason this +** might fail is due to an OOM fault. +*/ +static int jsonForceRCStr(JsonString *p){ + if( p->bStatic==0 ) return 1; + jsonGrow(p, p->nAlloc+1); + return p->bStatic==0; +} + /* Append N bytes from zIn onto the end of the JsonString string. */ static SQLITE_NOINLINE void jsonAppendExpand( @@ -588,6 +601,11 @@ static void jsonParseReset(JsonParse *pParse){ sqlite3_free(pParse->aUp); pParse->aUp = 0; } + if( pParse->zAlt ){ + char *z = pParse->zAlt; + pParse->zAlt = 0; + sqlite3RCStrUnref(z); + } if( pParse->bOwnsJson ){ /* Order operations so that if the destructor for pParse->zJson ** invokes jsonParseFree(), the recursion will terminate harmlessly */ @@ -650,7 +668,7 @@ static void jsonRenderNode( JsonString *pOut /* Write JSON here */ ){ assert( pNode!=0 ); - while( (pNode->jnFlags & JNODE_REPLACE)!=0 ){ + while( (pNode->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ u32 idx = (u32)(pNode - pParse->aNode); u32 i = pParse->iSubst; while( 1 /*exit-by-break*/ ){ @@ -722,13 +740,14 @@ static void jsonRenderNode( jsonAppendChar(pOut, '['); for(;;){ while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); jsonRenderNode(pParse, &pNode[j], pOut); } j += jsonNodeSize(&pNode[j]); } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pNode->eU==2 ); pNode = &pParse->aNode[pNode->u.iAppend]; j = 1; @@ -741,7 +760,7 @@ static void jsonRenderNode( jsonAppendChar(pOut, '{'); for(;;){ while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); jsonRenderNode(pParse, &pNode[j], pOut); jsonAppendChar(pOut, ':'); @@ -750,6 +769,7 @@ static void jsonRenderNode( j += 1 + jsonNodeSize(&pNode[j+1]); } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pNode->eU==2 ); pNode = &pParse->aNode[pNode->u.iAppend]; j = 1; @@ -766,7 +786,8 @@ static void jsonRenderNode( static void jsonReturnJson( JsonParse *pParse, /* The complete JSON */ JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx /* Return value for this function */ + sqlite3_context *pCtx, /* Return value for this function */ + int bGenerateAlt /* Also store the rendered text in zAlt */ ){ JsonString s; if( pParse->oom ){ @@ -776,6 +797,12 @@ static void jsonReturnJson( if( pParse->nErr==0 ){ jsonInit(&s, pCtx); jsonRenderNode(pParse, pNode, &s); + if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ + jsonAppendChar(&s, 0); + s.nUsed--; + pParse->zAlt = sqlite3RCStrRef(s.zBuf); + pParse->nAlt = s.nUsed; + } jsonResult(&s); sqlite3_result_subtype(pCtx, JSON_SUBTYPE); } @@ -965,7 +992,7 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - jsonReturnJson(pParse, pNode, pCtx); + jsonReturnJson(pParse, pNode, pCtx, 0); break; } } @@ -1079,6 +1106,8 @@ static int jsonParseAddSubstNode( pParse->aNode[idx].eU = 4; pParse->aNode[idx].u.iPrev = pParse->iSubst; pParse->iSubst = idx; + pParse->hasMod = 1; + pParse->useMod = 1; return idx; } @@ -1729,6 +1758,7 @@ static int jsonParse( int i; memset(pParse, 0, sizeof(*pParse)); if( zJson==0 ) return 1; +//printf("PARSE %s\n", zJson); pParse->zJson = zJson; pParse->bOwnsJson = bTakeJson; pParse->nJPRef = 1; @@ -1837,9 +1867,10 @@ static int jsonParseFindParents(JsonParse *pParse){ ** and so the caller must free it. */ static JsonParse *jsonParseCached( - sqlite3_context *pCtx, - sqlite3_value *pJson, - sqlite3_context *pErrCtx + sqlite3_context *pCtx, /* Context to use for cache search */ + sqlite3_value *pJson, /* Function param containing JSON text */ + sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ + int bUnedited /* No prior edits allowed */ ){ char *zJson = (char*)sqlite3_value_text(pJson); int nJson = sqlite3_value_bytes(pJson); @@ -1859,10 +1890,24 @@ static JsonParse *jsonParseCached( } if( pMatch==0 && p->nJson==nJson + && (p->hasMod==0 || bUnedited==0) && memcmp(p->zJson,zJson,nJson)==0 ){ p->nErr = 0; + p->useMod = 0; pMatch = p; +//printf("HIT %s at %d\n", zJson, iKey); + }else + if( pMatch==0 + && p->zAlt!=0 + && bUnedited==0 + && p->nAlt==nJson + && memcmp(p->zAlt, zJson, nJson)==0 + ){ + p->nErr = 0; + p->useMod = 1; + pMatch = p; +//printf("HIT %s at %d-alt\n", zJson, iKey); }else if( p->iHoldiHold; iMinKey = iKey; @@ -1872,11 +1917,18 @@ static JsonParse *jsonParseCached( } } if( pMatch ){ + /* The input JSON text was found in the cache. Use the preexisting + ** parse of this JSON */ pMatch->nErr = 0; pMatch->iHold = iMaxHold+1; assert( pMatch->nJPRef>0 ); /* pMatch is owned by the cache */ return pMatch; } + + /* The input JSON was not found anywhere in the cache. We will need + ** to parse it ourselves and generate a new JsonParse object. + */ +//printf("MISS %s\n", zJson); p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ){ sqlite3_result_error_nomem(pCtx); @@ -1954,7 +2006,7 @@ static JsonNode *jsonLookupStep( u32 i, j, nKey; const char *zKey; JsonNode *pRoot = &pParse->aNode[iRoot]; - while( (pRoot->jnFlags & JNODE_REPLACE)!=0 ){ + while( (pRoot->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ u32 idx = (u32)(pRoot - pParse->aNode); i = pParse->iSubst; while( 1 /*exit-by-break*/ ){ @@ -2004,6 +2056,7 @@ static JsonNode *jsonLookupStep( j += jsonNodeSize(&pRoot[j]); } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pRoot->eU==2 ); iRoot = pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; @@ -2012,6 +2065,7 @@ static JsonNode *jsonLookupStep( if( pApnd ){ u32 iStart, iLabel; JsonNode *pNode; + assert( pParse->useMod ); iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); zPath += i; @@ -2041,10 +2095,11 @@ static JsonNode *jsonLookupStep( if( pRoot->eType!=JSON_ARRAY ) return 0; for(;;){ while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; j += jsonNodeSize(&pBase[j]); } if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pBase->eU==2 ); iBase = pBase->u.iAppend; pBase = &pParse->aNode[iBase]; @@ -2074,11 +2129,14 @@ static JsonNode *jsonLookupStep( zPath += j + 1; j = 1; for(;;){ - while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; + while( j<=pRoot->n + && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE)!=0 && pParse->useMod)) + ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i--; j += jsonNodeSize(&pRoot[j]); } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pRoot->eU==2 ); iRoot = pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; @@ -2090,6 +2148,7 @@ static JsonNode *jsonLookupStep( if( i==0 && pApnd ){ u32 iStart; JsonNode *pNode; + assert( pParse->useMod ); iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); if( pParse->oom ) return 0; @@ -2392,7 +2451,7 @@ static void jsonArrayLengthFunc( u32 i; JsonNode *pNode; - p = jsonParseCached(ctx, argv[0], ctx); + p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; assert( p->nNode ); if( argc==2 ){ @@ -2405,9 +2464,14 @@ static void jsonArrayLengthFunc( return; } if( pNode->eType==JSON_ARRAY ){ - assert( (pNode->jnFlags & JNODE_APPEND)==0 ); - for(i=1; i<=pNode->n; n++){ - i += jsonNodeSize(&pNode[i]); + while( 1 /*exit-by-break*/ ){ + for(i=1; i<=pNode->n; n++){ + i += jsonNodeSize(&pNode[i]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( p->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &p->aNode[pNode->u.iAppend]; } } sqlite3_result_int64(ctx, n); @@ -2454,7 +2518,7 @@ static void jsonExtractFunc( JsonString jx; if( argc<2 ) return; - p = jsonParseCached(ctx, argv[0], ctx); + p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; if( argc==2 ){ /* With a single PATH argument */ @@ -2487,7 +2551,7 @@ static void jsonExtractFunc( } if( pNode ){ if( flags & JSON_JSON ){ - jsonReturnJson(p, pNode, ctx); + jsonReturnJson(p, pNode, ctx, 0); }else{ jsonReturn(p, pNode, ctx); sqlite3_result_subtype(ctx, 0); @@ -2613,12 +2677,14 @@ static void jsonPatchFunc( jsonParseReset(&x); return; } + x.useMod = 1; + y.useMod = 1; pResult = jsonMergePatch(&x, 0, y.aNode); assert( pResult!=0 || x.oom ); if( pResult && x.oom==0 ){ jsonDebugPrintParse(&x); jsonDebugPrintNode(pResult); - jsonReturnJson(&x, pResult, ctx); + jsonReturnJson(&x, pResult, ctx, 0); }else{ sqlite3_result_error_nomem(ctx); } @@ -2679,27 +2745,30 @@ static void jsonRemoveFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ + JsonParse *pParse; /* The parse */ JsonNode *pNode; const char *zPath; u32 i; if( argc<1 ) return; - if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return; - assert( x.nNode ); + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); + if( pParse==0 ) return; for(i=1; i<(u32)argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) goto remove_done; - pNode = jsonLookup(&x, zPath, 0, ctx); - if( x.nErr ) goto remove_done; - if( pNode ) pNode->jnFlags |= JNODE_REMOVE; + pNode = jsonLookup(pParse, zPath, 0, ctx); + if( pParse->nErr ) goto remove_done; + if( pNode ){ + pNode->jnFlags |= JNODE_REMOVE; + pParse->hasMod = 1; + pParse->useMod = 1; + } } - if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(&x, x.aNode, ctx); + if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){ + jsonReturnJson(pParse, pParse->aNode, ctx, 1); } remove_done: - jsonDebugPrintParse(&x); - jsonParseReset(&x); + jsonDebugPrintParse(p); } /* @@ -2763,7 +2832,7 @@ static void jsonReplaceNode( sqlite3_result_error_nomem(pCtx); } }else{ - JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx); + JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1); if( pPatch==0 ){ p->oom = 1; break; @@ -2817,7 +2886,7 @@ static void jsonReplaceFunc( jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]); } } - jsonReturnJson(&x, x.aNode, ctx); + jsonReturnJson(&x, x.aNode, ctx, 0); replace_err: jsonDebugPrintParse(&x); jsonParseReset(&x); @@ -2858,6 +2927,7 @@ static void jsonSetFunc( for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); bApnd = 0; + x.useMod = 1; pNode = jsonLookup(&x, zPath, &bApnd, ctx); if( x.oom ){ sqlite3_result_error_nomem(ctx); @@ -2869,7 +2939,7 @@ static void jsonSetFunc( } } jsonDebugPrintParse(&x); - jsonReturnJson(&x, x.aNode, ctx); + jsonReturnJson(&x, x.aNode, ctx, 0); jsonSetDone: jsonParseReset(&x); @@ -2891,7 +2961,7 @@ static void jsonTypeFunc( const char *zPath; JsonNode *pNode; - p = jsonParseCached(ctx, argv[0], ctx); + p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; if( argc==2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); @@ -2918,12 +2988,12 @@ static void jsonValidFunc( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv[0], 0); + p = jsonParseCached(ctx, argv[0], 0, 0); if( p==0 || p->oom ){ sqlite3_result_error_nomem(ctx); sqlite3_free(p); }else{ - sqlite3_result_int(ctx, p->nErr==0 && p->hasNonstd==0); + sqlite3_result_int(ctx, p->nErr==0 && (p->hasNonstd==0 || p->useMod)); if( p->nErr ) jsonParseFree(p); } } @@ -2964,7 +3034,7 @@ static void jsonErrorFunc( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv[0], 0); + p = jsonParseCached(ctx, argv[0], 0, 0); if( p==0 || p->oom ){ sqlite3_result_error_nomem(ctx); sqlite3_free(p); diff --git a/src/printf.c b/src/printf.c index 222e70be29..a55bd2b85b 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1406,6 +1406,7 @@ void sqlite3RCStrUnref(char *z){ } } +#if 0 /* ** Return true if the reference count on the string is exactly one, meaning ** that the string can be modified. Return false if the reference count @@ -1419,6 +1420,7 @@ int sqlite3RCStrIsWriteable(char *z){ assert( p->uMagic==SQLITE_RCSTR_MAGIC ); return p->nRCRef==1; } +#endif /* ** Create a new string that is capable of holding N bytes of text, not counting @@ -1441,6 +1443,7 @@ char *sqlite3RCStrNew(u64 N){ return (char*)&p[1]; } +#if 0 /* ** Return the number of bytes allocated to the string. The value returned ** does not include the space for the zero-terminator at the end. @@ -1456,6 +1459,7 @@ u64 sqlite3RCStrSize(char *z){ N -= sizeof(p) + 1; return N; } +#endif /* ** Change the size of the string so that it is able to hold N bytes. @@ -1477,6 +1481,7 @@ char *sqlite3RCStrResize(char *z, u64 N){ } } +#if 0 /* ** Add a new attachment to the string. ** @@ -1494,7 +1499,9 @@ void sqlite3RCStrAttach(char *z, void *pAttach, void(*xFree)(void*)){ p->xFree = xFree; p->pAttach = pAttach; } - +#endif + +#if 0 /* ** Return the attachment associated with a string if the attachment ** has the destructure specified in the second argument. If the @@ -1509,3 +1516,4 @@ void *sqlite3RCStrGetAttachment(char *z, void(*xFree)(void*)){ assert( p->uMagic==SQLITE_RCSTR_MAGIC ); return p->xFree==xFree ? p->pAttach : 0; } +#endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index adf4d34dec..358063534f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -5222,7 +5222,7 @@ void sqlite3FileSuffix3(const char*, char*); u8 sqlite3GetBoolean(const char *z,u8); const void *sqlite3ValueText(sqlite3_value*, u8); -int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*)); +//int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*)); int sqlite3ValueBytes(sqlite3_value*, u8); void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); @@ -5333,11 +5333,11 @@ int sqlite3OpenTempDatabase(Parse *); char *sqlite3RCStrRef(char*); void sqlite3RCStrUnref(char*); char *sqlite3RCStrNew(u64); -u64 sqlite3RCStrSize(char*); +//u64 sqlite3RCStrSize(char*); char *sqlite3RCStrResize(char*,u64); -int sqlite3RCStrIsWriteable(char*); -void sqlite3RCStrAttach(char*, void*, void(*)(void*)); -void *sqlite3RCStrGetAttachment(char*,void(*)(void*)); +//int sqlite3RCStrIsWriteable(char*); +//void sqlite3RCStrAttach(char*, void*, void(*)(void*)); +//void *sqlite3RCStrGetAttachment(char*,void(*)(void*)); void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); int sqlite3StrAccumEnlarge(StrAccum*, i64); diff --git a/src/vdbemem.c b/src/vdbemem.c index 87dfbbebd8..d44a41716f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1368,6 +1368,7 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return valueToText(pVal, enc); } +#if 0 /* Return true if sqlit3_value object pVal is a string or blob value ** that uses the destructor specified in the second argument. ** @@ -1385,6 +1386,7 @@ int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){ return 0; } } +#endif /* ** Create a new sqlite3_value object. From 44f53b96472a660e42f4c4f33e01f0fc9c691440 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Jul 2023 01:05:08 +0000 Subject: [PATCH 12/16] Extend the enhancement to json_set() and json_replace(). Clean up cruft. FossilOrigin-Name: 2dbb22c75e86f2e3ced38ac14b4943570d5c2f86cd5e37e875bf0c863be28836 --- manifest | 18 +++++------ manifest.uuid | 2 +- src/json.c | 44 ++++++++++++-------------- src/printf.c | 82 ------------------------------------------------- src/sqliteInt.h | 45 ++++++--------------------- src/vdbemem.c | 20 ------------ 6 files changed, 39 insertions(+), 172 deletions(-) diff --git a/manifest b/manifest index 875c317328..b712bbf97f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sJsonParse.zAlt\sfield\sto\sold\srevised\sJSON\stext\safter\sa\schange.\nDemonstrate\sthat\sthis\selminates\sthe\sneed\sfor\sreparsing\safter\sa\schange\nby\susing\sit\sin\sthe\sjson_remove()\sfunction.\s\sThis\sis\san\sincremental\scheck-in\ncontaining\slots\sof\scruft. -D 2023-07-26T00:48:45.768 +C Extend\sthe\senhancement\sto\sjson_set()\sand\sjson_replace().\s\sClean\sup\scruft. +D 2023-07-26T01:05:08.003 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a7823b0593eac42a608f8439e6b1f28022ac5fce9aa81a82856e48dd07ab601d +F src/json.c 790ca70b1f8884610fe5401ac6f260d955d47d9c14c9f3d2aa6541fd881faf07 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -634,7 +634,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1 -F src/printf.c 56e362bbefbe61b5b211f1886e3cdb9b25d8ba817c12df3a15f11b9bdc152efd +F src/printf.c e3ba080e2f409f9bfcc8d34724e6fc160e9c718dc92d0548f6b71b8b6f860ce2 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 @@ -643,7 +643,7 @@ F src/shell.c.in d320d8a13636de06d777cc1eab981caca304e175464e98183cf4ea68d93db81 F src/sqlite.h.in f999ef3642f381d69679b2516b430dbcb6c5a2a951b7f5e43dc4751b474a5774 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 29c8e6d0b6dfa9c2c356f23ab8349d7fac3679d5f7cfc01d4ee45c9171db2960 +F src/sqliteInt.h 30d7b0d586a4d03a384dcb60088c81b6fc6f74ce85cc3a0b3242eedc3cc24dbd F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -714,7 +714,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c 46ea371b10c3573cf9287d385411fbcc6cd42099ad3505d20e33f2f3ed91537d +F src/vdbemem.c 33da4f30ddba2670bc1e617c3262b66aef2a8039043d4ff93e5c97974991089d F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P afe02a398a16d51bd7482b6fbe2fbd15d9ac4fd9cdbc9d2bf81f38b3391fc567 -R 33c717c250039ffbe53dc8d7b2094929 +P f930b139d6db0ee799bc90397b225175103c4bf22923d1c9cbcd32509adc1738 +R 80113afdab5484e027b5180529a678b7 U drh -Z 96ad60bdf5cbde0ec2b901493a583f6e +Z dc859019c8db0487fd3e03bdb875d59d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dce1119850..4bbeaca0db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f930b139d6db0ee799bc90397b225175103c4bf22923d1c9cbcd32509adc1738 \ No newline at end of file +2dbb22c75e86f2e3ced38ac14b4943570d5c2f86cd5e37e875bf0c863be28836 \ No newline at end of file diff --git a/src/json.c b/src/json.c index d7d464442b..a4a3e65c92 100644 --- a/src/json.c +++ b/src/json.c @@ -1758,7 +1758,6 @@ static int jsonParse( int i; memset(pParse, 0, sizeof(*pParse)); if( zJson==0 ) return 1; -//printf("PARSE %s\n", zJson); pParse->zJson = zJson; pParse->bOwnsJson = bTakeJson; pParse->nJPRef = 1; @@ -1896,7 +1895,6 @@ static JsonParse *jsonParseCached( p->nErr = 0; p->useMod = 0; pMatch = p; -//printf("HIT %s at %d\n", zJson, iKey); }else if( pMatch==0 && p->zAlt!=0 @@ -1907,7 +1905,6 @@ static JsonParse *jsonParseCached( p->nErr = 0; p->useMod = 1; pMatch = p; -//printf("HIT %s at %d-alt\n", zJson, iKey); }else if( p->iHoldiHold; iMinKey = iKey; @@ -1928,7 +1925,6 @@ static JsonParse *jsonParseCached( /* The input JSON was not found anywhere in the cache. We will need ** to parse it ourselves and generate a new JsonParse object. */ -//printf("MISS %s\n", zJson); p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ){ sqlite3_result_error_nomem(pCtx); @@ -2866,7 +2862,7 @@ static void jsonReplaceFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ + JsonParse *pParse; /* The parse */ JsonNode *pNode; const char *zPath; u32 i; @@ -2876,20 +2872,20 @@ static void jsonReplaceFunc( jsonWrongNumArgs(ctx, "replace"); return; } - if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return; - assert( x.nNode ); + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); + if( pParse==0 ) return; for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); - pNode = jsonLookup(&x, zPath, 0, ctx); - if( x.nErr ) goto replace_err; + pParse->useMod = 1; + pNode = jsonLookup(pParse, zPath, 0, ctx); + if( pParse->nErr ) goto replace_err; if( pNode ){ - jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]); + jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - jsonReturnJson(&x, x.aNode, ctx, 0); + jsonReturnJson(pParse, pParse->aNode, ctx, 1); replace_err: - jsonDebugPrintParse(&x); - jsonParseReset(&x); + jsonDebugPrintParse(pParse); } @@ -2910,7 +2906,7 @@ static void jsonSetFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ + JsonParse *pParse; /* The parse */ JsonNode *pNode; const char *zPath; u32 i; @@ -2922,27 +2918,27 @@ static void jsonSetFunc( jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } - if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return; - assert( x.nNode ); + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); + if( pParse==0 ) return; for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); bApnd = 0; - x.useMod = 1; - pNode = jsonLookup(&x, zPath, &bApnd, ctx); - if( x.oom ){ + pParse->useMod = 1; + pNode = jsonLookup(pParse, zPath, &bApnd, ctx); + if( pParse->oom ){ sqlite3_result_error_nomem(ctx); goto jsonSetDone; - }else if( x.nErr ){ + }else if( pParse->nErr ){ goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ - jsonReplaceNode(ctx, &x, (u32)(pNode - x.aNode), argv[i+1]); + jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - jsonDebugPrintParse(&x); - jsonReturnJson(&x, x.aNode, ctx, 0); + jsonDebugPrintParse(pParse); + jsonReturnJson(pParse, pParse->aNode, ctx, 1); jsonSetDone: - jsonParseReset(&x); + /* no cleanup required */; } /* diff --git a/src/printf.c b/src/printf.c index a55bd2b85b..87ad91f795 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1394,34 +1394,13 @@ void sqlite3RCStrUnref(char *z){ assert( p!=0 ); p--; assert( p->nRCRef>0 ); - assert( p->uMagic==SQLITE_RCSTR_MAGIC ); if( p->nRCRef>=2 ){ p->nRCRef--; }else{ - if( p->xFree ) p->xFree(p->pAttach); -#ifdef SQLITE_DEBUG - p->uMagic = 0; -#endif sqlite3_free(p); } } -#if 0 -/* -** Return true if the reference count on the string is exactly one, meaning -** that the string can be modified. Return false if the reference count -** is greater than one. -*/ -int sqlite3RCStrIsWriteable(char *z){ - RCStr *p = (RCStr*)z; - assert( p!=0 ); - p--; - assert( p->nRCRef>0 ); - assert( p->uMagic==SQLITE_RCSTR_MAGIC ); - return p->nRCRef==1; -} -#endif - /* ** Create a new string that is capable of holding N bytes of text, not counting ** the zero byte at the end. The string is uninitialized. @@ -1435,32 +1414,9 @@ char *sqlite3RCStrNew(u64 N){ RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 ); if( p==0 ) return 0; p->nRCRef = 1; - p->xFree = 0; - p->pAttach = 0; -#ifdef SQLITE_DEBUG - p->uMagic = SQLITE_RCSTR_MAGIC; -#endif return (char*)&p[1]; } -#if 0 -/* -** Return the number of bytes allocated to the string. The value returned -** does not include the space for the zero-terminator at the end. -*/ -u64 sqlite3RCStrSize(char *z){ - RCStr *p = (RCStr*)z; - u64 N; - assert( p!=0 ); - p--; - assert( p->nRCRef>0 ); - assert( p->uMagic==SQLITE_RCSTR_MAGIC ); - N = sqlite3_msize(p); - N -= sizeof(p) + 1; - return N; -} -#endif - /* ** Change the size of the string so that it is able to hold N bytes. ** The string might be reallocated, so return the new allocation. @@ -1471,7 +1427,6 @@ char *sqlite3RCStrResize(char *z, u64 N){ assert( p!=0 ); p--; assert( p->nRCRef==1 ); - assert( p->uMagic==SQLITE_RCSTR_MAGIC ); pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1); if( pNew==0 ){ sqlite3_free(p); @@ -1480,40 +1435,3 @@ char *sqlite3RCStrResize(char *z, u64 N){ return (char*)&pNew[1]; } } - -#if 0 -/* -** Add a new attachment to the string. -** -** A string may have no more than one attachment. When a new attachment -** is added, any prior attachment is destroyed. Remove an attachment -** by adding a zero-attachment. -*/ -void sqlite3RCStrAttach(char *z, void *pAttach, void(*xFree)(void*)){ - RCStr *p = (RCStr*)z; - assert( p!=0 ); - p--; - assert( p->nRCRef>0 ); - assert( p->uMagic==SQLITE_RCSTR_MAGIC ); - if( p->xFree ) p->xFree(p->pAttach); - p->xFree = xFree; - p->pAttach = pAttach; -} -#endif - -#if 0 -/* -** Return the attachment associated with a string if the attachment -** has the destructure specified in the second argument. If the -** string has no attachment or if the destructor does not match, -** then return a NULL pointr. -*/ -void *sqlite3RCStrGetAttachment(char *z, void(*xFree)(void*)){ - RCStr *p = (RCStr*)z; - assert( p!=0 ); - p--; - assert( p->nRCRef>0 ); - assert( p->uMagic==SQLITE_RCSTR_MAGIC ); - return p->xFree==xFree ? p->pAttach : 0; -} -#endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 358063534f..a19a16d50d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4066,44 +4066,22 @@ struct sqlite3_str { ** The following object is the header for an "RCStr" or "reference-counted ** string". An RCStr is passed around and used like any other char* ** that has been dynamically allocated. The important interface -** difference is that it uses sqlite3RCStrUnref() as its destructor -** rather than sqlite3_free(). Other than that the two are interchangeable. +** differences: ** -** Thus to return an RCStr object as the result of an SQL function use: -** -** sqlite3_result_text64(ctx,z,sz,sqlite3RCStrUnref,SQLITE_UTF8) -** ^^^^^^^^^^^^^^^^^ -** Instead of sqlite3_free() or similar +** 1. RCStr strings are reference counted. They are deallocated +** when the reference count reaches zero. ** -** An SQL function can check its arguments to see if they are RCStr -** strings using the sqlite3ValueIsOfClass() function: +** 2. Use sqlite3RCStrUnref() to free an RCStr string rather than +** sqlite3_free() ** -** sqlite3ValueIsOfClass(argv[i], sqlite3RCStrUnref); -** -** An RCStr string might be better than an ordinary string in some cases -** because: -** -** (1) You can duplicate it using sqlite3RCStrRef(x). -** -** (2) You can also add an associated object to the string. For -** example, if the string is JSON, perhaps the associated object -** is a parse of that JSON. -** -** Methods for an RCStr string begin with "sqlite3RCStr...". +** 3. Make a (read-only) copy of a read-only RCStr string using +** sqlite3RCStrRef(). */ struct RCStr { - u32 nRCRef; /* Number of references */ -#ifdef SQLITE_DEBUG - u32 uMagic; /* Magic number for sanity checking */ -#endif - void *pAttach; /* Attachment to this string */ - void (*xFree)(void*); /* Destructor for the attachment */ + u64 nRCRef; /* Number of references */ + /* Total structure size should be a multiple of 8 bytes for alignment */ }; -/* The Magic number used by RCStr for sanity checking. SQLITE_DEBUG only. */ -#define SQLITE_RCSTR_MAGIC 0x3dc05d54 - - /* ** A pointer to this structure is used to communicate information ** from sqlite3Init and OP_ParseSchema into the sqlite3InitCallback. @@ -5222,7 +5200,6 @@ void sqlite3FileSuffix3(const char*, char*); u8 sqlite3GetBoolean(const char *z,u8); const void *sqlite3ValueText(sqlite3_value*, u8); -//int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*)); int sqlite3ValueBytes(sqlite3_value*, u8); void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); @@ -5333,11 +5310,7 @@ int sqlite3OpenTempDatabase(Parse *); char *sqlite3RCStrRef(char*); void sqlite3RCStrUnref(char*); char *sqlite3RCStrNew(u64); -//u64 sqlite3RCStrSize(char*); char *sqlite3RCStrResize(char*,u64); -//int sqlite3RCStrIsWriteable(char*); -//void sqlite3RCStrAttach(char*, void*, void(*)(void*)); -//void *sqlite3RCStrGetAttachment(char*,void(*)(void*)); void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); int sqlite3StrAccumEnlarge(StrAccum*, i64); diff --git a/src/vdbemem.c b/src/vdbemem.c index d44a41716f..c73e59c362 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1368,26 +1368,6 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return valueToText(pVal, enc); } -#if 0 -/* Return true if sqlit3_value object pVal is a string or blob value -** that uses the destructor specified in the second argument. -** -** TODO: Maybe someday promote this interface into a published API so -** that third-party extensions can get access to it? -*/ -int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){ - if( ALWAYS(pVal!=0) - && (pVal->flags & (MEM_Str|MEM_Blob))!=0 - && (pVal->flags & MEM_Dyn)!=0 - && pVal->xDel==xFree - ){ - return 1; - }else{ - return 0; - } -} -#endif - /* ** Create a new sqlite3_value object. */ From 9a184daad3991239595b007e1bb2b1167aeb2cac Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Jul 2023 11:00:47 +0000 Subject: [PATCH 13/16] Fix jsonForceRCStr() to also add the NULL terminator. FossilOrigin-Name: 134b01f37f8f741d7f7b7eda81384695d1cbe4c39751d87f08832d5c9afdcef2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 28 ++++++++++++++++------------ 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 44f7e7b86b..e2bd372d36 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\sfixes\sinto\sthe\sjson-opt\sbranch. -D 2023-07-26T01:15:34.571 +C Fix\sjsonForceRCStr()\sto\salso\sadd\sthe\sNULL\sterminator. +D 2023-07-26T11:00:47.983 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 790ca70b1f8884610fe5401ac6f260d955d47d9c14c9f3d2aa6541fd881faf07 +F src/json.c b3f20f01dc6e11071b0b5a8fda73a16c23fd7e65356ec715c0082e904f3fb284 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4c1af616e672a0d4d04f2652e645617758231598fb7161b956883512303ae87 54b3c43fdfdaca6b129a5f0ee93c34eb001663775d33c087066650f5e164d1c1 -R 31a2d8c82324ae8c8feb0c4037e19a24 +P ef4e1664d159b98854d9aa580b0bb942f1726f32a190e2ea659c171131ddce9a +R d6f55ea1e7a815bfa0f96bc129102acf U drh -Z df17bf2cf88062fccaa06a3f8068b84c +Z d5c0e742c545064eb8b7fe7e17a2bad7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 80d363ba5c..02d918d3db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef4e1664d159b98854d9aa580b0bb942f1726f32a190e2ea659c171131ddce9a \ No newline at end of file +134b01f37f8f741d7f7b7eda81384695d1cbe4c39751d87f08832d5c9afdcef2 \ No newline at end of file diff --git a/src/json.c b/src/json.c index a4a3e65c92..3ad5af659c 100644 --- a/src/json.c +++ b/src/json.c @@ -248,16 +248,6 @@ static int jsonGrow(JsonString *p, u32 N){ return SQLITE_OK; } -/* Try to force the string to be an RCStr string, rather than a -** static string. Return true on success. The only reason this -** might fail is due to an OOM fault. -*/ -static int jsonForceRCStr(JsonString *p){ - if( p->bStatic==0 ) return 1; - jsonGrow(p, p->nAlloc+1); - return p->bStatic==0; -} - /* Append N bytes from zIn onto the end of the JsonString string. */ static SQLITE_NOINLINE void jsonAppendExpand( @@ -315,6 +305,22 @@ static void jsonAppendChar(JsonString *p, char c){ } } +/* Try to force the string to be a zero-terminated RCStr string. +** +** Return true on success. Return false if an OOM prevents this +** from happening. +*/ +static int jsonForceRCStr(JsonString *p){ + jsonAppendChar(p, 0); + if( p->bErr ) return 0; + p->nUsed--; + if( p->bStatic==0 ) return 1; + p->nAlloc = 0; + jsonGrow(p, p->nUsed); + return p->bStatic==0; +} + + /* Append a comma separator to the output buffer, if the previous ** character is not '[' or '{'. */ @@ -798,8 +804,6 @@ static void jsonReturnJson( jsonInit(&s, pCtx); jsonRenderNode(pParse, pNode, &s); if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ - jsonAppendChar(&s, 0); - s.nUsed--; pParse->zAlt = sqlite3RCStrRef(s.zBuf); pParse->nAlt = s.nUsed; } From 7725370ff42e5be18aa97e480e9b8379167da692 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Jul 2023 11:43:39 +0000 Subject: [PATCH 14/16] Improved comments and other cleanup for the changes on this branch. FossilOrigin-Name: bac953a80d1a541e7a12aef00c86c002133859237143ad670b39ea19799a8900 --- manifest | 12 ++-- manifest.uuid | 2 +- src/json.c | 174 +++++++++++++++++++++++++------------------------- 3 files changed, 93 insertions(+), 95 deletions(-) diff --git a/manifest b/manifest index e2bd372d36..ad979b08e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sjsonForceRCStr()\sto\salso\sadd\sthe\sNULL\sterminator. -D 2023-07-26T11:00:47.983 +C Improved\scomments\sand\sother\scleanup\sfor\sthe\schanges\son\sthis\sbranch. +D 2023-07-26T11:43:39.965 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c b3f20f01dc6e11071b0b5a8fda73a16c23fd7e65356ec715c0082e904f3fb284 +F src/json.c 15ca3c2dfd5a525c225da5592f66d4ef6b789a83814f417be0a0af617071bc38 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef4e1664d159b98854d9aa580b0bb942f1726f32a190e2ea659c171131ddce9a -R d6f55ea1e7a815bfa0f96bc129102acf +P 134b01f37f8f741d7f7b7eda81384695d1cbe4c39751d87f08832d5c9afdcef2 +R 9df4f7348f2973215055bf0d567d169d U drh -Z d5c0e742c545064eb8b7fe7e17a2bad7 +Z b486171855fdc62c1515867c409e89e4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 02d918d3db..989660b794 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -134b01f37f8f741d7f7b7eda81384695d1cbe4c39751d87f08832d5c9afdcef2 \ No newline at end of file +bac953a80d1a541e7a12aef00c86c002133859237143ad670b39ea19799a8900 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3ad5af659c..8fe3116bcf 100644 --- a/src/json.c +++ b/src/json.c @@ -59,7 +59,7 @@ static const char jsonIsSpace[] = { typedef struct JsonString JsonString; typedef struct JsonNode JsonNode; typedef struct JsonParse JsonParse; -typedef struct JsonTask JsonTask; +typedef struct JsonCleanup JsonCleanup; /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator @@ -75,11 +75,11 @@ struct JsonString { char zSpace[100]; /* Initial static space */ }; -/* A deferred cleanup task. A list of JsonTask objects might be +/* A deferred cleanup task. A list of JsonCleanup objects might be ** run when the JsonParse object is destroyed. */ -struct JsonTask { - JsonTask *pJTNext; /* Next in a list */ +struct JsonCleanup { + JsonCleanup *pJCNext; /* Next in a list */ void (*xOp)(void*); /* Routine to run */ void *pArg; /* Argument to xOp() */ }; @@ -136,29 +136,50 @@ struct JsonNode { }; -/* A completely parsed JSON string +/* A parsed and possibly edited JSON string. Lifecycle: +** +** 1. JSON comes in and is parsed into an array aNode[]. The original +** JSON text is stored in zJson. This object may or may not be the +** owner of the input JSON - the bOwnsJson variables determines which. +** +** 2. Zero or more changes are made (via json_remove() or json_replace() +** or similar) to the aNode[] array. +** +** 3. A new, edited and mimified JSON string is generated from aNode +** and stored in zAlt. The JsonParse object always owns zAlt. +** +** Step 1 always happens. Step 2 and 3 may or may not happen, depending +** on the operation. +** +** aNode[].u.zJContent entries typically point into zJson. Hence zJson +** must remain valid for the lifespan of the parse. For edits, +** aNode[].u.zJContent might point to malloced space other than zJson. +** Entries in pClup are responsible for freeing that extra malloced space. +** +** When walking the parse tree in aNode[], edits are ignored if useMod is +** false. */ struct JsonParse { u32 nNode; /* Number of slots of aNode[] used */ u32 nAlloc; /* Number of slots of aNode[] allocated */ JsonNode *aNode; /* Array of nodes containing the parse */ char *zJson; /* Original JSON string (before edits) */ - char *zAlt; /* Revised JSON after edits applied. Maybe NULL */ + char *zAlt; /* Revised and/or mimified JSON */ u32 *aUp; /* Index of parent of each node */ - JsonTask *pClean; /* Cleanup operations prior to freeing this object */ + JsonCleanup *pClup;/* Cleanup operations prior to freeing this object */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ - u8 nJPRef; /* Number of references to this object */ u8 bOwnsJson; /* This object owns zJson and response for freeing it */ u8 useMod; /* Actually use the edits contain inside aNode */ u8 hasMod; /* aNode contains edits from the original zJson */ + u32 nJPRef; /* Number of references to this object */ int nJson; /* Length of the zJson string in bytes */ int nAlt; /* Length of alternative JSON string zAlt, in bytes */ u32 iErr; /* Error location in zJson[] */ - u32 iSubst; /* Last known JSON_SUBST node */ - u32 iHold; /* Replace cache line with the lowest iHold value */ + u32 iSubst; /* Last JSON_SUBST entry in aNode[] */ + u32 iHold; /* Age of this entry in the cache for LRU replacement */ }; /* @@ -174,17 +195,6 @@ struct JsonParse { ** Utility routines for dealing with JsonString objects **************************************************************************/ -#if 0 -/* -** This is a destructor for JSON strings. We make it a separate function -** so that the sqlite3ValueIsOfClass() function can be used to unambiguously -** identify sqlite3_value objects that are known JSON strings. -*/ -static void jsonStringClass(void *p){ - sqlite3RCStrUnref((char*)p); -} -#endif - /* Set the JsonString object to an empty string */ static void jsonZero(JsonString *p){ @@ -202,7 +212,6 @@ static void jsonInit(JsonString *p, sqlite3_context *pCtx){ jsonZero(p); } - /* Free all allocated memory and reset the JsonString object back to its ** initial state. */ @@ -211,8 +220,7 @@ static void jsonReset(JsonString *p){ jsonZero(p); } - -/* Report an out-of-memory (OOM) condition +/* Report an out-of-memory (OOM) condition */ static void jsonOom(JsonString *p){ p->bErr = 1; @@ -316,7 +324,9 @@ static int jsonForceRCStr(JsonString *p){ p->nUsed--; if( p->bStatic==0 ) return 1; p->nAlloc = 0; + p->nUsed++; jsonGrow(p, p->nUsed); + p->nUsed--; return p->bStatic==0; } @@ -392,7 +402,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendRawNZ(p, zIn, i); zIn += i; N -= i; - if( N==0 ) break; + if( N==0 ) break; } assert( zIn[0]=='\\' ); switch( (u8)zIn[1] ){ @@ -501,7 +511,7 @@ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ /* -** Append a function parameter value to the JSON string under +** Append a function parameter value to the JSON string under ** construction. */ static void jsonAppendValue( @@ -550,18 +560,11 @@ static void jsonAppendValue( ** The JSON string is reset. */ static void jsonResult(JsonString *p){ - if( p->bErr==0 ){ - if( p->bStatic ){ - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, - SQLITE_UTF8); - }else{ - jsonAppendChar(p, 0); - p->nUsed--; - sqlite3RCStrRef(p->zBuf); - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, - (void(*)(void*))sqlite3RCStrUnref, - SQLITE_UTF8); - } + if( p->bErr==0 && jsonForceRCStr(p) ){ + sqlite3RCStrRef(p->zBuf); + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + (void(*)(void*))sqlite3RCStrUnref, + SQLITE_UTF8); }else if( p->bErr==1 ){ sqlite3_result_error_nomem(p->pCtx); } @@ -590,9 +593,9 @@ static u32 jsonNodeSize(JsonNode *pNode){ ** delete the JsonParse object itself. */ static void jsonParseReset(JsonParse *pParse){ - while( pParse->pClean ){ - JsonTask *pTask = pParse->pClean; - pParse->pClean = pTask->pJTNext; + while( pParse->pClup ){ + JsonCleanup *pTask = pParse->pClup; + pParse->pClup = pTask->pJCNext; pTask->xOp(pTask->pArg); sqlite3_free(pTask); } @@ -608,17 +611,13 @@ static void jsonParseReset(JsonParse *pParse){ pParse->aUp = 0; } if( pParse->zAlt ){ - char *z = pParse->zAlt; + sqlite3RCStrUnref(pParse->zAlt); pParse->zAlt = 0; - sqlite3RCStrUnref(z); } if( pParse->bOwnsJson ){ - /* Order operations so that if the destructor for pParse->zJson - ** invokes jsonParseFree(), the recursion will terminate harmlessly */ - char *z = pParse->zJson; - pParse->zJson = 0; pParse->bOwnsJson = 0; - sqlite3RCStrUnref(z); + sqlite3RCStrUnref(pParse->zJson); + pParse->zJson = 0; } } @@ -650,14 +649,14 @@ static int jsonParseAddCleanup( void(*xOp)(void*), /* The cleanup task */ void *pArg /* Argument to the cleanup */ ){ - JsonTask *pTask = sqlite3_malloc64( sizeof(*pTask) ); + JsonCleanup *pTask = sqlite3_malloc64( sizeof(*pTask) ); if( pTask==0 ){ pParse->oom = 1; xOp(pArg); return SQLITE_ERROR; } - pTask->pJTNext = pParse->pClean; - pParse->pClean = pTask; + pTask->pJCNext = pParse->pClup; + pParse->pClup = pTask; pTask->xOp = xOp; pTask->pArg = pArg; return SQLITE_OK; @@ -870,8 +869,7 @@ static void jsonReturn( int rc; int bNeg = 0; const char *z; - - + assert( pNode->eU==1 ); z = pNode->u.zJContent; if( z[0]=='-' ){ z++; bNeg = 1; } @@ -1458,23 +1456,23 @@ json_parse_restart: cDelim = z[i]; for(j=i+1; 1; j++){ static const char aOk[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }; if( aOk[(unsigned char)z[j]] ) continue; c = z[j]; @@ -1740,18 +1738,18 @@ json_parse_restart: /* ** Parse a complete JSON string. Return 0 on success or non-zero if there -** are any errors. If an error occurs, free all memory associated with -** pParse. +** are any errors. If an error occurs, free all memory held by pParse, +** but not pParse itself. ** ** pParse is uninitialized when this routine is called. ** -** pParse->nJPRef set to 1. The caller becomes the owner of the +** pParse->nJPRef set to 1. The caller becomes the owner of the ** the JsonParse object. ** -** pParse->bOwnsJson is to bTakeJson. If bTakeJson is 1, the newly initialized -** JsonParse object will become the own the zJson input string. If bTakeJson -** is 0, then the caller is responsible for preserving zJson for the lifetime -** of the JsonParse object. +** pParse->bOwnsJson is set to bTakeJson. If bTakeJson is 1, the newly +** initialized JsonParse object will become the owner of the zJson input +** string. If bTakeJson is 0, then the caller is responsible for +** preserving zJson for the lifetime of the JsonParse object. */ static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ @@ -2129,7 +2127,7 @@ static JsonNode *jsonLookupStep( zPath += j + 1; j = 1; for(;;){ - while( j<=pRoot->n + while( j<=pRoot->n && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE)!=0 && pParse->useMod)) ){ if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i--; @@ -2255,7 +2253,7 @@ static void jsonWrongNumArgs( char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", zFuncName); sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); + sqlite3_free(zMsg); } /* @@ -2391,7 +2389,7 @@ static void jsonTest1Func( /* ** Implementation of the json_QUOTE(VALUE) function. Return a JSON value -** corresponding to the SQL value input. Mostly this means putting +** corresponding to the SQL value input. Mostly this means putting ** double-quotes around strings and returning the unquoted string "null" ** when given a NULL input. */ @@ -2438,7 +2436,7 @@ static void jsonArrayFunc( ** json_array_length(JSON) ** json_array_length(JSON, PATH) ** -** Return the number of elements in the top-level JSON array. +** Return the number of elements in the top-level JSON array. ** Return 0 if the input is not a well-formed JSON array. */ static void jsonArrayLengthFunc( @@ -2752,7 +2750,7 @@ static void jsonRemoveFunc( if( argc<1 ) return; pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); - if( pParse==0 ) return; + if( pParse==0 ) return; for(i=1; i<(u32)argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) goto remove_done; @@ -2854,7 +2852,7 @@ static void jsonReplaceNode( } } } - + /* ** json_replace(JSON, PATH, VALUE, ...) ** @@ -3269,7 +3267,7 @@ static int jsonEachConnect( UNUSED_PARAMETER(argv); UNUSED_PARAMETER(argc); UNUSED_PARAMETER(pAux); - rc = sqlite3_declare_vtab(db, + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," "json HIDDEN,root HIDDEN)"); if( rc==SQLITE_OK ){ @@ -3481,7 +3479,7 @@ static int jsonEachColumn( break; } case JEACH_ID: { - sqlite3_result_int64(ctx, + sqlite3_result_int64(ctx, (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); break; } @@ -3581,8 +3579,8 @@ static int jsonEachBestIndex( idxMask |= iMask; } } - if( pIdxInfo->nOrderBy>0 - && pIdxInfo->aOrderBy[0].iColumn<0 + if( pIdxInfo->nOrderBy>0 + && pIdxInfo->aOrderBy[0].iColumn<0 && pIdxInfo->aOrderBy[0].desc==0 ){ pIdxInfo->orderByConsumed = 1; @@ -3782,10 +3780,10 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_parse, 1, 0, jsonParseFunc), JFUNCTION(json_test1, 1, 0, jsonTest1Func), #endif - WAGGREGATE(json_group_array, 1, 0, 0, + WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), - WAGGREGATE(json_group_object, 2, 0, 0, + WAGGREGATE(json_group_object, 2, 0, 0, jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC) }; From 2b5719706647a6831e2d5fadd56d04f67fc63837 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Jul 2023 11:53:14 +0000 Subject: [PATCH 15/16] More comment improvements in json.c. Do not run jsonLookup() following an OOM error. FossilOrigin-Name: cd5fda8c2e354da7458b7c1a82ff18c5946f8dab16095bb0293bec57f6804f17 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 11 +++++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index ad979b08e4..490099b0ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\sand\sother\scleanup\sfor\sthe\schanges\son\sthis\sbranch. -D 2023-07-26T11:43:39.965 +C More\scomment\simprovements\sin\sjson.c.\s\sDo\snot\srun\sjsonLookup()\sfollowing\nan\sOOM\serror. +D 2023-07-26T11:53:14.698 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 15ca3c2dfd5a525c225da5592f66d4ef6b789a83814f417be0a0af617071bc38 +F src/json.c c784b923a8aaa5cb514c135a74d77aec1b5c9e1cbd63764adb10d44cc678bb6e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 134b01f37f8f741d7f7b7eda81384695d1cbe4c39751d87f08832d5c9afdcef2 -R 9df4f7348f2973215055bf0d567d169d +P bac953a80d1a541e7a12aef00c86c002133859237143ad670b39ea19799a8900 +R 968cb2bde9a2ab88d10bb801ca2ea188 U drh -Z b486171855fdc62c1515867c409e89e4 +Z 8edd1e3712b3e1a50841b71b7e4dc709 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 989660b794..979f3c9801 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bac953a80d1a541e7a12aef00c86c002133859237143ad670b39ea19799a8900 \ No newline at end of file +cd5fda8c2e354da7458b7c1a82ff18c5946f8dab16095bb0293bec57f6804f17 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 8fe3116bcf..5deeb6acba 100644 --- a/src/json.c +++ b/src/json.c @@ -1841,9 +1841,10 @@ static int jsonParseFindParents(JsonParse *pParse){ #define JSON_CACHE_SZ 4 /* Max number of cache entries */ /* -** Obtain a complete parse of the JSON found in the first argument -** of the argv array. Use the sqlite3_get_auxdata() cache for this -** parse if it is available. If the cache is not available or if it +** Obtain a complete parse of the JSON found in the pJson argument +** +** Use the sqlite3_get_auxdata() cache to find a preexisting parse +** if it is available. If the cache is not available or if it ** is no longer valid, parse the JSON again and return the new parse. ** Also register the new parse so that it will be available for ** future sqlite3_get_auxdata() calls. @@ -2003,7 +2004,9 @@ static JsonNode *jsonLookupStep( ){ u32 i, j, nKey; const char *zKey; - JsonNode *pRoot = &pParse->aNode[iRoot]; + JsonNode *pRoot; + if( pParse->oom ) return 0; + pRoot = &pParse->aNode[iRoot]; while( (pRoot->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ u32 idx = (u32)(pRoot - pParse->aNode); i = pParse->iSubst; From 7286c598963bcec929f44c3a75ff0b257acaf1cd Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Jul 2023 13:17:43 +0000 Subject: [PATCH 16/16] Change the debugging "json_parse(X)" function so that it shows a more complete description of the JsonParse object on standard output and returns the mimified JSON. Former behavior was to return the text of a decode of the aNode array. FossilOrigin-Name: 1bf85d4e388714a88f8940dcdec353c3e0267456697eff6963d34637912aecc9 --- manifest | 14 ++++---- manifest.uuid | 2 +- src/json.c | 80 ++++++++++++++++++++++++---------------------- test/fuzzdata6.db | Bin 1785856 -> 1785856 bytes 4 files changed, 50 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 490099b0ad..f0b7ab99fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\scomment\simprovements\sin\sjson.c.\s\sDo\snot\srun\sjsonLookup()\sfollowing\nan\sOOM\serror. -D 2023-07-26T11:53:14.698 +C Change\sthe\sdebugging\s"json_parse(X)"\sfunction\sso\sthat\sit\sshows\sa\smore\ncomplete\sdescription\sof\sthe\sJsonParse\sobject\son\sstandard\soutput\sand\sreturns\nthe\smimified\sJSON.\s\sFormer\sbehavior\swas\sto\sreturn\sthe\stext\sof\sa\sdecode\nof\sthe\saNode\sarray. +D 2023-07-26T13:17:43.116 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c c784b923a8aaa5cb514c135a74d77aec1b5c9e1cbd63764adb10d44cc678bb6e +F src/json.c 512bc389b42c68b34571ca532afb0f4bb235b1fc11ea5c9a4f6850c64fa12ab4 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -1131,7 +1131,7 @@ F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5b F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba F test/fuzzdata4.db b502c7d5498261715812dd8b3c2005bad08b3a26e6489414bd13926cd3e42ed2 F test/fuzzdata5.db e35f64af17ec48926481cfaf3b3855e436bd40d1cfe2d59a9474cb4b748a52a5 -F test/fuzzdata6.db 92a80e4afc172c24f662a10a612d188fb272de4a9bd19e017927c95f737de6d7 +F test/fuzzdata6.db b8725a5f5cf7a3b7241a9038e57ca7e7cc8c3f4d86b44bd770617bda245ab2b0 F test/fuzzdata7.db 0166b56fd7a6b9636a1d60ef0a060f86ddaecf99400a666bb6e5bbd7199ad1f2 F test/fuzzdata8.db f6c2f2af4deaaae0ddb3310d509c2659990794aa653dc501b80a0534c3493f80 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bac953a80d1a541e7a12aef00c86c002133859237143ad670b39ea19799a8900 -R 968cb2bde9a2ab88d10bb801ca2ea188 +P cd5fda8c2e354da7458b7c1a82ff18c5946f8dab16095bb0293bec57f6804f17 +R ecfa3d900e8af899b83f6d4944c0b5ca U drh -Z 8edd1e3712b3e1a50841b71b7e4dc709 +Z 621bef24d346a4651dc554beab6ec135 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 979f3c9801..fa90aa429f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd5fda8c2e354da7458b7c1a82ff18c5946f8dab16095bb0293bec57f6804f17 \ No newline at end of file +1bf85d4e388714a88f8940dcdec353c3e0267456697eff6963d34637912aecc9 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 5deeb6acba..9eb2302684 100644 --- a/src/json.c +++ b/src/json.c @@ -118,7 +118,12 @@ static const char * const jsonType[] = { #define JNODE_JSON5 0x40 /* Node contains JSON5 enhancements */ -/* A single node of parsed JSON +/* A single node of parsed JSON. An array of these nodes describes +** a parse of JSON + edits. +** +** Use the json_parse() SQL function (available when compiled with +** -DSQLITE_DEBUG) to see a dump of complete JsonParse objects, including +** a complete listing and decoding of the array of JsonNodes. */ struct JsonNode { u8 eType; /* One of the JSON_ type values */ @@ -2283,7 +2288,7 @@ static void jsonRemoveAllNulls(JsonNode *pNode){ ** SQL functions used for testing and debugging ****************************************************************************/ -#if 0 /* One for debugging. zero normally */ +#if SQLITE_DEBUG /* ** Print N node entries. */ @@ -2299,12 +2304,15 @@ static void jsonDebugPrintNodeEntries( }else{ zType = jsonType[aNode[i].eType]; } + printf("node %4u: %-7s n=%-5d", i, zType, aNode[i].n); if( (aNode[i].jnFlags & ~JNODE_LABEL)!=0 ){ - printf("node %4u: %-7s %02x n=%-5d", - i, zType, aNode[i].jnFlags, aNode[i].n); - }else{ - printf("node %4u: %-7s n=%-5d", - i, zType, aNode[i].n); + u8 f = aNode[i].jnFlags; + if( f & JNODE_RAW ) printf(" RAW"); + if( f & JNODE_ESCAPE ) printf(" ESCAPE"); + if( f & JNODE_REMOVE ) printf(" REMOVE"); + if( f & JNODE_REPLACE ) printf(" REPLACE"); + if( f & JNODE_APPEND ) printf(" APPEND"); + if( f & JNODE_JSON5 ) printf(" JSON5"); } switch( aNode[i].eU ){ case 1: printf(" zJContent=[%.*s]\n", @@ -2316,6 +2324,10 @@ static void jsonDebugPrintNodeEntries( } } } +#endif /* SQLITE_DEBUG */ + + +#if 0 /* 1 for debugging. 0 normally. Requires -DSQLITE_DEBUG too */ static void jsonDebugPrintParse(JsonParse *p){ jsonDebugPrintNodeEntries(p->aNode, p->nNode); } @@ -2330,45 +2342,37 @@ static void jsonDebugPrintNode(JsonNode *pNode){ #ifdef SQLITE_DEBUG /* -** The json_parse(JSON) function returns a string which describes -** a parse of the JSON provided. Or it returns NULL if JSON is not -** well-formed. +** SQL function: json_parse(JSON) +** +** Parse JSON using jsonParseCached(). Then print a dump of that +** parse on standard output. Return the mimified JSON result, just +** like the json() function. */ static void jsonParseFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv ){ - JsonString s; /* Output string - not real JSON */ - JsonParse x; /* The parse */ - u32 i; + JsonParse *p; /* The parse */ assert( argc==1 ); - if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return; - jsonParseFindParents(&x); - jsonInit(&s, ctx); - for(i=0; inNode); + printf("nAlloc = %u\n", p->nAlloc); + printf("nJson = %d\n", p->nJson); + printf("nAlt = %d\n", p->nAlt); + printf("nErr = %u\n", p->nErr); + printf("oom = %u\n", p->oom); + printf("hasNonstd = %u\n", p->hasNonstd); + printf("bOwnsJson = %u\n", p->bOwnsJson); + printf("useMod = %u\n", p->useMod); + printf("hasMod = %u\n", p->hasMod); + printf("nJPRef = %u\n", p->nJPRef); + printf("iSubst = %u\n", p->iSubst); + printf("iHold = %u\n", p->iHold); + jsonDebugPrintNodeEntries(p->aNode, p->nNode); + jsonReturnJson(p, p->aNode, ctx, 1); } /* diff --git a/test/fuzzdata6.db b/test/fuzzdata6.db index b1424c21e4194f352629d4d08b7b88d16ea0d1ee..67076d0026aca74aa3984260ecd3371006966501 100644 GIT binary patch delta 2927 zcmaJ@Yj6|S72dmAy;l3sO4e(IWVtndfUVlbp*##&1{?#Kj0vQZX#%xD0|Cp#U~rpE zhRVik5}C?Y82d1W7(srd8Ik}Ya7&sH!wexI)7GR++mJ93)1*wBbjBG<(hPmw)e3uX zSNz#O-*>)y&iT&0XXEkqc)WekkJ!|VrAC$tZ~IkuREZ)bN-I%=5@nRAQHh$As9A|x zPDiaT9<~M7Q4-xj748Ub#9Z3CI>=KLWn!ovo6^Nly>K!~M_^1rC2+VMm0ChuJD=KC zR$UHPhAf*P{-K4#JKnVpjL`+R!4PLF&>dVAXD=48no76Sw-d%j*ir}$u?#~3E1~<2 z#R=^rZ1djK5Q|eowp}ZOC#XP7zUqy*UUmBI3Gp+rM(D8}v)nRO89JyF`-?Kqz-Y$i zFe`P5I)+L>Yl@o#qkp!2i?;{44g_B{@*z`QV&T2m!Ov}j$h;)6fT8*gLlLe<3u(P+ zOM7|G+Z0F5rvi;~pXcgnfqUJ)$D1D?>+69pBqGmxrs$^Zk(GtncYr=zbxe3EjIZU#iRdFflm25%*DBza<0?8 zU5Td{>UlUn&W*xkU|CK$z7f|3vN3e5ZkyT)*EIg zvkEDviZYuKR*Lu^S#qjMU4L~hzfqVd<`3Wn^6_nfMDAKF$h(RU@_G0b5DbRjNs$X93^*Q z*Z;%zl|q=V-G{f&lFBt%AhN%KYh~O*NWWqGSvD5`>mkADw8;rlq9a#gi?Ca^6MH$* zlJ=#KERQJReZFj=&AWsppmAWTids54*LKj`RO$-H(~E(MYe+ zCI+M)sX?`Ofu016|5CEz*dA%U+UfcEDaCz+LR^|R%_U9X)90jCWU5L2kf4xSU*N`} z20=ik#Z!X>EiXwHJls#rCMV6+J&91vcy&^0Q7I4TQnDjsO=*MD)x*sI6TIiyv>?v8CTfxtazFCvYX6r8fm&& zHEjZca8xsGxt<^Qad7S%C&URI@a7Oq+>_9tGFGt_jrZ zp_%|XsR?L{u6f$Ca^jRGuvkw36ZCEg^OB2jeL||#o(vp1D{WV&qfRewa+*z(H*ev6 zc}ue92JO5nM4n!G@*-jTm}Ga|v#^sZrj!<|3tN5F$==GnF-=qGxVKncHwn*2F4?hGJj65X3UiYA16}+w85$ab z8=NL`(FeAkx!^28jG7}Hd7m$bmOajE>c;}V`;>nKsy^b&@t>m3a=*5x4cW5EKZCP( z`Vt$65v?+$?E|%7va<$7^)?K&O+y~fj$BD!+S!*b=u5lKq}?b)TVDdhn7a(Uf-A<{ zKeY%K-+T9Sv;%J^T+Qh*)J(45~$$18E^SLzTKWL+)V|?oubqwUKbwZ z4zXWbmzk5s8%!O25((t5FN1}M_qz42ZYewg>4&@@!aJMT`Q9Bc}7&H(?Y ueCc$7{My|gq3$&YXTyzJ5;){*Fl8b!R&|>uk_8#`j_^{vKTKqp;w#Og< delta 3034 zcmai$X>e0j6vy99UbDR|H?PS{UPzjbLR%li<%xG&=;>r5g z&Fu*t!+<_yDMi1MvXJG?d{c`O-M}*8DbPQ`=7x1$7QuY!>_lj}Xp*6Eucdx#XSaoP zc5@pSdV3kq8uc@0wf$MyZEcgjktT~_?vUw%VZ1)b^mY{{H$!idvw>g~iy@)%zZueE z$lxH?f%G+Pv)~_tICV&<|646|P1{UZd(*H`!#OVQ6zIR^nHh{{rn*6S^@PYDQP_yT zZ7vkP$)_^?+~eZ&0H5db!P+kQNc7 zo;tOI@JV5dN^RKDND2qUBE*iRtq18m=SX{y7%MD6~r# z8{q?Dv#O8yA16P(FO{GyqNXBvY%}&!QO=yj2o+y}pzKlX(Fz%(A+g6N8JyJSYIVEv ziQQ#ukPh-cS!Nhp*mrbT^Xvj5I4tcJsQo*2q_9G&M__$RIzGg{JzE=gNu*(y^{K;Z zfbj&?Q_ezr#P*g|6zVOzP22K5);pO%S9z!k0#%qNB?Dps)e*&p%N-ENi`DpEr1!_J zN;i3UcvUUl=XO#*=tL-|?LOOm(oz0*OO8eIs}0Of=?HlFdEXl85ht> z<8v}86H0Q1s}Own8(*x86=z5GHeb>EK#x1+zAlnlCf5R!PBkMDa4Mk<+s_GKKKlaRbs%k4*z> zo9#NNKg^vO)^CcLNMc92-BWWFGm?!l9MC4BA}d0Co#|oqXAP%^>dUTliYYl6Gnu;w z8!k=O{FqHoDRw*?(_>tAN14?5R$iOFJL?L-ylta zj$QJfU^~yrmp1_VupK4z>)XCs+m$AdAY}Pw6phQ#cz!f)KN)u*Uur{4IEvAeq%`4p z(9E{n-#r0t3@baijMWl+?>LH}_pGA;d>sxe>3-L-W@GATl~PKNmQ^WDEZb`AaYJX9 z;|{25a;$(KPdP16e?d8#x#NtVuYB>xM7r3>NMyWA$Gq%TT+PaNj=KEkWC?GQ&Wa28 zeU=~0wZ>T9AUj)k42g8fOcJr$dDd)fod|cbOcVxBD-}>b$=RNfdTU~llOz_os+XjM z#a-9y7cwh5tL?Bd?iyrU8us1>54`5umT`n6`(1_0WU6K?^Ab~VwYSdGGzQ$g>KiBW m)x)Z(<@P5guuSVTKrS`RXZTSJgUznGt$l8l^ts&)SN{tsuf1ge