1
0
mirror of https://github.com/sqlite/sqlite.git synced 2025-11-09 14:21:03 +03:00

Fix up the JSON cache to work better.

FossilOrigin-Name: 1fdbc39521f63aedc6f08ecaafa54ea467b8c6316a692a18ad01eecbf22a0977
This commit is contained in:
drh
2023-12-01 18:46:14 +00:00
parent 5bfa7e65d1
commit 063d0d4c3a
3 changed files with 109 additions and 109 deletions

View File

@@ -1,5 +1,5 @@
C Cache\sis\sworking\sbetter,\sbut\sdoes\snot\spreserve\sthe\shasJson5\sflag. C Fix\sup\sthe\sJSON\scache\sto\swork\sbetter.
D 2023-12-01T13:28:13.897 D 2023-12-01T18:46:14.485
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
F src/json.c 32e3741e5ff9f8380ebf84dcd6d3c28ddcd7f6a2923b72a05165ad108ca0c278 F src/json.c 3556e879386b0af1c542ed13ebb2f58c48d3b8b3ab13a96ab77ad2a9493cc715
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9
@@ -2145,11 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 25ed295f300fea6185104a73721076bccd2b2a6e411c78564266fa6dca4ff70c P a12add7ab9f5aee5bb2ede0c4d22e599dd28f7a107dce72b2ea48ef92d233e8a
R b145e990ca1c690b2dc7d50ed4bd4abc R bed96169a434a08ba8168cff3e09a739
T *branch * jsonb-cache
T *sym-jsonb-cache *
T -sym-jsonb *
U drh U drh
Z 663fd17309f10179421e079add21165c Z 599839251e392336311bcdbf8af02666
# Remove this line to create a well-formed Fossil manifest. # Remove this line to create a well-formed Fossil manifest.

View File

@@ -1 +1 @@
a12add7ab9f5aee5bb2ede0c4d22e599dd28f7a107dce72b2ea48ef92d233e8a 1fdbc39521f63aedc6f08ecaafa54ea467b8c6316a692a18ad01eecbf22a0977

View File

@@ -201,7 +201,6 @@ static const char jsonIsOk[256] = {
/* Objects */ /* Objects */
typedef struct JsonCache JsonCache; typedef struct JsonCache JsonCache;
typedef struct JsonCacheLine JsonCacheLine;
typedef struct JsonString JsonString; typedef struct JsonString JsonString;
typedef struct JsonParse JsonParse; typedef struct JsonParse JsonParse;
@@ -217,16 +216,10 @@ typedef struct JsonParse JsonParse;
** All content, both JSON text and the JSONB blobs, is stored as RCStr ** All content, both JSON text and the JSONB blobs, is stored as RCStr
** objects. ** objects.
*/ */
struct JsonCacheLine {
u32 nJson; /* Size of the JSON text, in bytes */
u32 nBlob; /* Size of the corresponding JSONB, in bytes */
char *zJson; /* RCStr holding the JSON text */
char *aBlob; /* RCStr holding the corresponding JSONB */
};
struct JsonCache { struct JsonCache {
sqlite3 *db; /* Database connection */ sqlite3 *db; /* Database connection */
int nUsed; /* Number of active entries in the cache */ int nUsed; /* Number of active entries in the cache */
JsonCacheLine a[JSON_CACHE_SIZE]; /* One line for each cache entry */ JsonParse *a[JSON_CACHE_SIZE]; /* One line for each cache entry */
}; };
/* An instance of this object represents a JSON string /* An instance of this object represents a JSON string
@@ -285,13 +278,14 @@ struct JsonParse {
u32 nBlob; /* Bytes of aBlob[] actually used */ u32 nBlob; /* Bytes of aBlob[] actually used */
u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */
char *zJson; /* Json text used for parsing */ char *zJson; /* Json text used for parsing */
int nJson; /* Length of the zJson string in bytes */
u16 iDepth; /* Nesting depth */ u16 iDepth; /* Nesting depth */
u8 nErr; /* Number of errors seen */ u8 nErr; /* Number of errors seen */
u8 oom; /* Set to true if out of memory */ u8 oom; /* Set to true if out of memory */
u8 bBlobIsRCStr; /* True if aBlob is an RCStr */ u8 bJsonIsRCStr; /* True if zJson is an RCStr */
u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */
u8 bReadOnly; /* Do not modify. */
u32 nJPRef; /* Number of references to this object */ u32 nJPRef; /* Number of references to this object */
int nJson; /* Length of the zJson string in bytes */
u32 iErr; /* Error location in zJson[] */ u32 iErr; /* Error location in zJson[] */
/* Search and edit information. See jsonLookupBlobStep() */ /* Search and edit information. See jsonLookupBlobStep() */
u8 eEdit; /* Edit operation to apply */ u8 eEdit; /* Edit operation to apply */
@@ -330,6 +324,9 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson);
static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*);
static void jsonReturnParse(sqlite3_context*,JsonParse*); static void jsonReturnParse(sqlite3_context*,JsonParse*);
static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32);
static void jsonParseFree(JsonParse*);
/************************************************************************** /**************************************************************************
** Utility routines for dealing with JsonCache objects ** Utility routines for dealing with JsonCache objects
**************************************************************************/ **************************************************************************/
@@ -340,8 +337,7 @@ static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32);
static void jsonCacheDelete(JsonCache *p){ static void jsonCacheDelete(JsonCache *p){
int i; int i;
for(i=0; i<p->nUsed; i++){ for(i=0; i<p->nUsed; i++){
sqlite3RCStrUnref(p->a[i].zJson); jsonParseFree(p->a[i]);
sqlite3RCStrUnref(p->a[i].aBlob);
} }
sqlite3DbFree(p->db, p); sqlite3DbFree(p->db, p);
} }
@@ -358,16 +354,12 @@ static void jsonCacheDeleteGeneric(void *p){
*/ */
static int jsonCacheInsert( static int jsonCacheInsert(
sqlite3_context *ctx, /* The SQL statement context holding the cache */ sqlite3_context *ctx, /* The SQL statement context holding the cache */
char *zJson, /* The key. */ JsonParse *pParse /* The parse object to be added to the cache */
u32 nJson, /* Number of bytes in zJson */
int bJsonIsRCStr, /* True if zJson is already an RCStr */
u8 *aBlob, /* The value. */
u32 nBlob /* Number of bytes in aBlob */
){ ){
JsonCache *p; JsonCache *p;
char *aRCBlob;
char *zRCJson;
assert( pParse->zJson!=0 );
assert( pParse->bJsonIsRCStr );
p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID);
if( p==0 ){ if( p==0 ){
sqlite3 *db = sqlite3_context_db_handle(ctx); sqlite3 *db = sqlite3_context_db_handle(ctx);
@@ -378,76 +370,61 @@ static int jsonCacheInsert(
p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID);
if( p==0 ) return SQLITE_NOMEM; if( p==0 ) return SQLITE_NOMEM;
} }
aRCBlob = sqlite3RCStrNew( nBlob );
if( aRCBlob==0 ) return SQLITE_NOMEM;
memcpy(aRCBlob, aBlob, nBlob);
if( bJsonIsRCStr ){
zRCJson = sqlite3RCStrRef(zJson);
}else{
zRCJson = sqlite3RCStrNew( nJson );
if( zRCJson==0 ){
sqlite3RCStrUnref(aRCBlob);
return SQLITE_NOMEM;
}
memcpy(zRCJson, zJson, nJson);
zRCJson[nJson] = 0;
}
if( p->nUsed >= JSON_CACHE_SIZE ){ if( p->nUsed >= JSON_CACHE_SIZE ){
sqlite3RCStrUnref(p->a[0].zJson); jsonParseFree(p->a[0]);
sqlite3RCStrUnref(p->a[0].aBlob);
memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0])); memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0]));
p->nUsed = JSON_CACHE_SIZE-1; p->nUsed = JSON_CACHE_SIZE-1;
} }
p->a[p->nUsed].nJson = nJson; pParse->eEdit = 0;
p->a[p->nUsed].nBlob = nBlob; pParse->nJPRef++;
p->a[p->nUsed].zJson = zRCJson; pParse->bReadOnly = 1;
p->a[p->nUsed].aBlob = aRCBlob; p->a[p->nUsed] = pParse;
p->nUsed++; p->nUsed++;
return SQLITE_OK; return SQLITE_OK;
} }
/* /*
** Search for a cached translation of zJson (size: nJson bytes) into ** Search for a cached translation the json text supplied by pArg. Return
** JSONB. Return it if found. ** the JsonParse object if found.
**
** The returned value is an RCStr object if it is not NULL.
** The caller is responsible for incrementing the reference count.
*/ */
static u8 *jsonCacheSearch( static JsonParse *jsonCacheSearch(
sqlite3_context *ctx, /* The SQL statement context holding the cache */ sqlite3_context *ctx, /* The SQL statement context holding the cache */
char *zJson, /* The key. Might or might not be an RCStr */ sqlite3_value *pArg /* Function argument containing SQL text */
u32 nJson, /* Size of the key in bytes */
u32 *pnBlob /* OUT: Size of the result in bytes */
){ ){
JsonCache *p; JsonCache *p;
int i; int i;
const char *zJson;
int nJson;
if( sqlite3_value_type(pArg)!=SQLITE_TEXT ){
return 0;
}
zJson = (const char*)sqlite3_value_text(pArg);
if( zJson==0 ) return 0;
nJson = sqlite3_value_bytes(pArg);
assert( pnBlob!=0 );
p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID);
if( p==0 ){ if( p==0 ){
*pnBlob = 0;
return 0; return 0;
} }
for(i=0; i<p->nUsed; i++){ for(i=0; i<p->nUsed; i++){
if( p->a[i].zJson==zJson ) break; if( p->a[i]->zJson==zJson ) break;
} }
if( i>=p->nUsed ){ if( i>=p->nUsed ){
for(i=0; i<p->nUsed; i++){ for(i=0; i<p->nUsed; i++){
if( p->a[i].nJson!=nJson ) continue; if( p->a[i]->nJson!=nJson ) continue;
if( memcmp(p->a[i].zJson, zJson, nJson)==0 ) break; if( memcmp(p->a[i]->zJson, zJson, nJson)==0 ) break;
} }
} }
if( i<p->nUsed ){ if( i<p->nUsed ){
if( i<p->nUsed-1 ){ if( i<p->nUsed-1 ){
JsonCacheLine tmp = p->a[i]; JsonParse *tmp = p->a[i];
memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp)); memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp));
p->a[p->nUsed-1] = tmp; p->a[p->nUsed-1] = tmp;
i = p->nUsed - 1; i = p->nUsed - 1;
} }
*pnBlob = p->a[i].nBlob; return p->a[i];
return (u8*)p->a[i].aBlob;
}else{ }else{
*pnBlob = 0;
return 0; return 0;
} }
} }
@@ -717,9 +694,9 @@ static void jsonAppendSqlValue(
** **
** The JsonString is reset. ** The JsonString is reset.
** **
** If pParse and ctx are both non-NULL and if pParse->aBlob is valid ** If pParse and ctx are both non-NULL, then the SQL string in p is
** then an attempt is made to cache the translation from JSON text into ** loaded into the zJson field of the pParse object as a RCStr and the
** the blob. ** pParse is added to the cache.
*/ */
static void jsonReturnString( static void jsonReturnString(
JsonString *p, /* String to return */ JsonString *p, /* String to return */
@@ -736,18 +713,19 @@ static void jsonReturnString(
sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed,
SQLITE_TRANSIENT, SQLITE_UTF8); SQLITE_TRANSIENT, SQLITE_UTF8);
}else if( jsonForceRCStr(p) ){ }else if( jsonForceRCStr(p) ){
sqlite3RCStrRef(p->zBuf); if( pParse && pParse->bJsonIsRCStr==0 ){
if( pParse ){ int rc;
int rc = jsonCacheInsert(ctx, p->zBuf, p->nUsed, 1, pParse->zJson = sqlite3RCStrRef(p->zBuf);
pParse->aBlob, pParse->nBlob); pParse->nJson = p->nUsed;
pParse->bJsonIsRCStr = 1;
rc = jsonCacheInsert(ctx, pParse);
if( rc==SQLITE_NOMEM ){ if( rc==SQLITE_NOMEM ){
sqlite3RCStrUnref(p->zBuf);
sqlite3_result_error_nomem(ctx); sqlite3_result_error_nomem(ctx);
jsonStringReset(p); jsonStringReset(p);
return; return;
} }
} }
sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, sqlite3_result_text64(p->pCtx, sqlite3RCStrRef(p->zBuf), p->nUsed,
sqlite3RCStrUnref, sqlite3RCStrUnref,
SQLITE_UTF8); SQLITE_UTF8);
}else{ }else{
@@ -771,12 +749,11 @@ static void jsonReturnString(
*/ */
static void jsonParseReset(JsonParse *pParse){ static void jsonParseReset(JsonParse *pParse){
assert( pParse->nJPRef<=1 ); assert( pParse->nJPRef<=1 );
if( pParse->bBlobIsRCStr ){ if( pParse->bJsonIsRCStr ){
assert( pParse->nBlobAlloc==0 ); sqlite3RCStrUnref(pParse->zJson);
sqlite3RCStrUnref((char*)pParse->aBlob); pParse->zJson = 0;
pParse->aBlob = 0; pParse->nJson = 0;
pParse->nBlob = 0; pParse->bJsonIsRCStr = 0;
pParse->bBlobIsRCStr = 0;
} }
if( pParse->nBlobAlloc ){ if( pParse->nBlobAlloc ){
sqlite3_free(pParse->aBlob); sqlite3_free(pParse->aBlob);
@@ -1048,6 +1025,7 @@ static int jsonBlobExpand(JsonParse *pParse, u32 N){
static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){ static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){
u8 *aOld; u8 *aOld;
u32 nSize; u32 nSize;
assert( !pParse->bReadOnly );
if( pParse->nBlobAlloc>0 ) return 1; if( pParse->nBlobAlloc>0 ) return 1;
aOld = pParse->aBlob; aOld = pParse->aBlob;
nSize = pParse->nBlob + nExtra; nSize = pParse->nBlob + nExtra;
@@ -2671,6 +2649,11 @@ jsonInsertIntoBlob_patherror:
return; return;
} }
/*
** Make a copy of a JsonParse object. The copy will be editable.
*/
/* /*
** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob,
** from the SQL function argument pArg. Return a pointer to the new ** from the SQL function argument pArg. Return a pointer to the new
@@ -2692,17 +2675,40 @@ static JsonParse *jsonParseFuncArg(
sqlite3_value *pArg, sqlite3_value *pArg,
u32 flgs u32 flgs
){ ){
int eType; /* Datatype of pArg */ int eType; /* Datatype of pArg */
JsonParse *p = 0; /* Value to be returned */ JsonParse *p = 0; /* Value to be returned */
JsonParse *pFromCache = 0; /* Value taken from cache */
assert( ctx!=0 ); assert( ctx!=0 );
eType = sqlite3_value_type(pArg); eType = sqlite3_value_type(pArg);
if( eType==SQLITE_NULL ){ if( eType==SQLITE_NULL ){
return 0; return 0;
} }
pFromCache = jsonCacheSearch(ctx, pArg);
if( pFromCache ){
pFromCache->nJPRef++;
if( (flgs & JSON_EDITABLE)==0 ){
return pFromCache;
}
}
rebuild_from_cache:
p = sqlite3_malloc64( sizeof(*p) ); p = sqlite3_malloc64( sizeof(*p) );
if( p==0 ) goto json_pfa_oom; if( p==0 ) goto json_pfa_oom;
memset(p, 0, sizeof(*p)); memset(p, 0, sizeof(*p));
p->nJPRef = 1;
if( pFromCache!=0 ){
u32 nBlob = pFromCache->nBlob;
p->aBlob = sqlite3_malloc64( nBlob );
if( p->aBlob==0 ) goto json_pfa_oom;
memcpy(p->aBlob, pFromCache->aBlob, nBlob);
p->nBlobAlloc = p->nBlob = nBlob;
p->hasNonstd = pFromCache->hasNonstd;
jsonParseFree(pFromCache);
return p;
}else{
jsonParseFree(pFromCache);
pFromCache = 0;
}
if( eType==SQLITE_BLOB ){ if( eType==SQLITE_BLOB ){
u32 n, sz = 0; u32 n, sz = 0;
p->aBlob = (u8*)sqlite3_value_blob(pArg); p->aBlob = (u8*)sqlite3_value_blob(pArg);
@@ -2729,29 +2735,11 @@ static JsonParse *jsonParseFuncArg(
} }
return p; return p;
} }
/* TODO: Check in the cache */
p->zJson = (char*)sqlite3_value_text(pArg); p->zJson = (char*)sqlite3_value_text(pArg);
p->nJson = sqlite3_value_bytes(pArg); p->nJson = sqlite3_value_bytes(pArg);
if( p->nJson==0 ) goto json_pfa_malformed; if( p->nJson==0 ) goto json_pfa_malformed;
if( p->zJson==0 ) goto json_pfa_oom; if( p->zJson==0 ) goto json_pfa_oom;
if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){
p->aBlob = jsonCacheSearch(ctx, p->zJson, p->nJson, &p->nBlob);
if( p->aBlob ){
if( flgs & JSON_EDITABLE ){
u8 *pNew = sqlite3_malloc64( p->nBlob );
if( pNew==0 ) goto json_pfa_oom;
memcpy(pNew, p->aBlob, p->nBlob);
p->aBlob = pNew;
p->nBlobAlloc = p->nBlob;
}else{
sqlite3RCStrRef((char*)p->aBlob);
p->bBlobIsRCStr = 1;
}
return p;
}
if( flgs & JSON_KEEPERROR ) ctx = 0;
if( jsonConvertTextToBlob(p, ctx) ){
if( flgs & JSON_KEEPERROR ){ if( flgs & JSON_KEEPERROR ){
p->nErr = 1; p->nErr = 1;
return p; return p;
@@ -2759,12 +2747,26 @@ static JsonParse *jsonParseFuncArg(
jsonParseFree(p); jsonParseFree(p);
return 0; return 0;
} }
} }else{
if( ctx ){
int isRCStr = sqlite3ValueIsOfClass(pArg, sqlite3RCStrUnref); int isRCStr = sqlite3ValueIsOfClass(pArg, sqlite3RCStrUnref);
int rc = jsonCacheInsert(ctx, p->zJson, p->nJson, isRCStr, int rc;
p->aBlob, p->nBlob); if( !isRCStr ){
char *zNew = sqlite3RCStrNew( p->nJson );
if( zNew==0 ) goto json_pfa_oom;
memcpy(zNew, p->zJson, p->nJson);
p->zJson = zNew;
p->zJson[p->nJson] = 0;
}else{
sqlite3RCStrRef(p->zJson);
}
p->bJsonIsRCStr = 1;
rc = jsonCacheInsert(ctx, p);
if( rc==SQLITE_NOMEM ) goto json_pfa_oom; if( rc==SQLITE_NOMEM ) goto json_pfa_oom;
if( flgs & JSON_EDITABLE ){
pFromCache = p;
p = 0;
goto rebuild_from_cache;
}
} }
return p; return p;
@@ -2779,6 +2781,7 @@ json_pfa_malformed:
} }
json_pfa_oom: json_pfa_oom:
jsonParseFree(pFromCache);
jsonParseFree(p); jsonParseFree(p);
sqlite3_result_error_nomem(ctx); sqlite3_result_error_nomem(ctx);
return 0; return 0;