1
0
mirror of https://github.com/sqlite/sqlite.git synced 2025-11-09 14:21:03 +03:00

First attempt to get the JSON text-to-binary cache working. All test cases

pass, but the cache seems not to help much.

FossilOrigin-Name: 25ed295f300fea6185104a73721076bccd2b2a6e411c78564266fa6dca4ff70c
This commit is contained in:
drh
2023-12-01 12:57:12 +00:00
parent 4b9ed1b256
commit ca1ce7773c
5 changed files with 215 additions and 75 deletions

View File

@@ -1,5 +1,5 @@
C Remove\sall\strace\sof\sJsonNode\sfrom\sthe\sJSON\simplementation.\s\sThe\sJSONB\sformat\nis\sused\sas\sthe\sinternal\sbinary\sencoding\sfor\ssearching\sand\sediting.
D 2023-11-30T23:36:14.577
C First\sattempt\sto\sget\sthe\sJSON\stext-to-binary\scache\sworking.\s\sAll\stest\scases\npass,\sbut\sthe\scache\sseems\snot\sto\shelp\smuch.
D 2023-12-01T12:57:12.578
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
F src/json.c c4f3602115334c210fbcdfb9565ba9fd322d29f7327b598fc0dd5dea4cb1b068
F src/json.c 9a96faf916f572deb94b6cc7673ba6e299c48d153c18218e104a2c6ae37c2c8e
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9
@@ -724,7 +724,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
F src/pragma.c b5b4cff830575e6188cd56a295a57448d2b9dbc53f0dae58e22b97354cda3781
F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c
F src/printf.c 9da63b9ae1c14789bcae12840f5d800fd9302500cd2d62733fac77f0041b4750
F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640
F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916
F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
@@ -733,7 +733,7 @@ F src/shell.c.in 7bb83293775e1a5586d65212997442bc7acc70a2f1b781745da64ec3c2e4ea9
F src/sqlite.h.in d93a4821d2f792467a60f7dc81268d1bb8634f40c31694ef254cab4f9921f96a
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
F src/sqliteInt.h 6b82eb99a9d2887e873fb29e56befb7c50cf4624df615d23a28f071dc8abf5f6
F src/sqliteInt.h aab66d149269f15f6f1011081b389f001f00b84045c69a4f9ec96dd68cc3a7d7
F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 7b5756fa6d00b093bf083a8d7a5ef5485f7a09e4eac473785c8380688f861a1b
R 38620fda20feff15c621a1f3bd3d0184
P 11ebb5f712cc7a515e2e0f2be8c1d71de20c97fe5b74c4f4d72c84fd21182d35
R 18f6bb3dc7a73112aa5c773b9941447e
U drh
Z 66786c8d6e89beb8e06f2e4708cb9069
Z 9a005b4c0b238db6966807971445cc41
# Remove this line to create a well-formed Fossil manifest.

View File

@@ -1 +1 @@
11ebb5f712cc7a515e2e0f2be8c1d71de20c97fe5b74c4f4d72c84fd21182d35
25ed295f300fea6185104a73721076bccd2b2a6e411c78564266fa6dca4ff70c

View File

@@ -165,7 +165,7 @@ static const char jsonIsSpace[] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
#define fast_isspace(x) (jsonIsSpace[(unsigned char)x])
#define jsonIsspace(x) (jsonIsSpace[(unsigned char)x])
/*
** Characters that are special to JSON. Control charaters,
@@ -200,9 +200,35 @@ static const char jsonIsOk[256] = {
#endif
/* Objects */
typedef struct JsonCache JsonCache;
typedef struct JsonCacheLine JsonCacheLine;
typedef struct JsonString JsonString;
typedef struct JsonParse JsonParse;
/*
** Magic number used for the JSON parse cache in sqlite3_get_auxdata()
*/
#define JSON_CACHE_ID (-429938) /* Cache entry */
#define JSON_CACHE_SIZE 4 /* Max number of cache entries */
/* A cache mapping JSON text into JSONB blobs.
**
** All content, both JSON text and the JSONB blobs, is stored as RCStr
** objects.
*/
struct JsonCacheLine {
u32 nJson; /* Size of the JSON text, in bytes */
u32 nBlob; /* Size of the corresponding JSONB, in bytes */
char *zJson; /* RCStr holding the JSON text */
char *aBlob; /* RCStr holding the corresponding JSONB */
};
struct JsonCache {
sqlite3 *db; /* Database connection */
int nUsed; /* Number of active entries in the cache */
JsonCacheLine a[JSON_CACHE_SIZE]; /* One line for each cache entry */
};
/* An instance of this object represents a JSON string
** under construction. Really, this is a generic string accumulator
** that can be and is used to create strings other than JSON.
@@ -258,11 +284,11 @@ struct JsonParse {
u8 *aBlob; /* JSONB representation of JSON value */
u32 nBlob; /* Bytes of aBlob[] actually used */
u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */
char *zJson; /* Original JSON string (before edits) */
char *zJson; /* Json text used for parsing */
u16 iDepth; /* Nesting depth */
u8 nErr; /* Number of errors seen */
u8 oom; /* Set to true if out of memory */
u8 bJsonIsRCStr; /* True if zJson is an RCStr */
u8 bBlobIsRCStr; /* True if aBlob is an RCStr */
u8 hasNonstd; /* True if input uses non-standard features like JSON5 */
u32 nJPRef; /* Number of references to this object */
int nJson; /* Length of the zJson string in bytes */
@@ -304,6 +330,113 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson);
static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*);
static void jsonReturnParse(sqlite3_context*,JsonParse*);
static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32);
/**************************************************************************
** Utility routines for dealing with JsonCache objects
**************************************************************************/
/*
** Free a JsonCache object.
*/
static void jsonCacheDelete(JsonCache *p){
int i;
for(i=0; i<p->nUsed; i++){
sqlite3RCStrUnref(p->a[i].zJson);
sqlite3RCStrUnref(p->a[i].aBlob);
}
sqlite3DbFree(p->db, p);
}
static void jsonCacheDeleteGeneric(void *p){
jsonCacheDelete((JsonCache*)p);
}
/*
** Insert a new entry into the cache. If the cache is full, expell
** the least recently used entry. Return SQLITE_OK on success or a
** result code otherwise.
**
** Both the input JSON and JSONB must be RCStr objects.
*/
static int jsonCacheInsert(
sqlite3_context *ctx, /* The SQL statement context holding the cache */
char *zJson, /* The key. Must be an RCStr! */
u32 nJson, /* Number of bytes in zJson */
char *aBlob, /* The value. Not an RCStr */
u32 nBlob /* Number of bytes in aBlob */
){
JsonCache *p;
char *aRCBlob = 0;
p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID);
if( p==0 ){
sqlite3 *db = sqlite3_context_db_handle(ctx);
p = sqlite3DbMallocZero(db, sizeof(*p));
if( p==0 ) return SQLITE_NOMEM;
p->db = db;
sqlite3_set_auxdata(ctx, JSON_CACHE_ID, p, jsonCacheDeleteGeneric);
p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID);
if( p==0 ) return SQLITE_NOMEM;
}
aRCBlob = sqlite3RCStrNew( nBlob );
if( aRCBlob==0 ) return SQLITE_NOMEM;
memcpy(aRCBlob, aBlob, nBlob);
if( p->nUsed >= JSON_CACHE_SIZE ){
sqlite3RCStrUnref(p->a[0].zJson);
sqlite3RCStrUnref(p->a[0].aBlob);
memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0]));
p->nUsed = JSON_CACHE_SIZE-1;
}
p->a[p->nUsed].nJson = nJson;
p->a[p->nUsed].nBlob = nBlob;
p->a[p->nUsed].zJson = sqlite3RCStrRef(zJson);
p->a[p->nUsed].aBlob = aRCBlob;
p->nUsed++;
return SQLITE_OK;
}
/*
** Search for a cached translation of zJson (size: nJson bytes) into
** JSONB. Return it if found.
**
** The returned value is an RCStr object if it is not NULL.
** The caller is responsible for incrementing the reference count.
*/
static u8 *jsonCacheSearch(
sqlite3_context *ctx, /* The SQL statement context holding the cache */
char *zJson, /* The key. Might or might not be an RCStr */
u32 nJson, /* Size of the key in bytes */
u32 *pnBlob /* OUT: Size of the result in bytes */
){
JsonCache *p;
int i;
assert( pnBlob!=0 );
p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID);
if( p==0 ){
*pnBlob = 0;
return 0;
}
for(i=0; i<p->nUsed; i++){
if( p->a[i].zJson==zJson ) break;
}
if( i>=p->nUsed ){
for(i=0; i<p->nUsed; i++){
if( p->a[i].nJson!=nJson ) continue;
if( memcmp(p->a[i].zJson, zJson, nJson)==0 ) break;
}
}
if( i<p->nUsed ){
if( i<p->nUsed-1 ){
JsonCacheLine tmp = p->a[i];
memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp));
p->a[p->nUsed-1] = tmp;
}
*pnBlob = p->a[i].nBlob;
return (u8*)p->a[i].aBlob;
}else{
*pnBlob = 0;
return 0;
}
}
/**************************************************************************
** Utility routines for dealing with JsonString objects
@@ -569,8 +702,18 @@ static void jsonAppendSqlValue(
** the result of the SQL function.
**
** The JsonString is reset.
**
** If pParse and ctx are both non-NULL and if pParse->aBlob is valid
** then an attempt is made to cache the translation from JSON text into
** the blob.
*/
static void jsonReturnString(JsonString *p){
static void jsonReturnString(
JsonString *p, /* String to return */
JsonParse *pParse, /* JSONB source or NULL */
sqlite3_context *ctx /* Where to cache */
){
assert( (pParse!=0)==(ctx!=0) );
assert( ctx==0 || ctx==p->pCtx );
if( p->eErr==0 ){
int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx));
if( flags & JSON_BLOB ){
@@ -580,6 +723,16 @@ static void jsonReturnString(JsonString *p){
SQLITE_TRANSIENT, SQLITE_UTF8);
}else if( jsonForceRCStr(p) ){
sqlite3RCStrRef(p->zBuf);
if( pParse ){
int rc = jsonCacheInsert(ctx, p->zBuf, p->nUsed,
(char*)pParse->aBlob, pParse->nBlob);
if( rc==SQLITE_NOMEM ){
sqlite3RCStrUnref(p->zBuf);
sqlite3_result_error_nomem(ctx);
jsonStringReset(p);
return;
}
}
sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed,
sqlite3RCStrUnref,
SQLITE_UTF8);
@@ -604,10 +757,12 @@ static void jsonReturnString(JsonString *p){
*/
static void jsonParseReset(JsonParse *pParse){
assert( pParse->nJPRef<=1 );
if( pParse->bJsonIsRCStr ){
sqlite3RCStrUnref(pParse->zJson);
pParse->zJson = 0;
pParse->bJsonIsRCStr = 0;
if( pParse->bBlobIsRCStr ){
assert( pParse->nBlobAlloc==0 );
sqlite3RCStrUnref((char*)pParse->aBlob);
pParse->aBlob = 0;
pParse->nBlob = 0;
pParse->bBlobIsRCStr = 0;
}
if( pParse->nBlobAlloc ){
sqlite3_free(pParse->aBlob);
@@ -636,46 +791,18 @@ static void jsonParseFree(JsonParse *pParse){
}
}
/*
** Translate a single byte of Hex into an integer.
** This routine only works if h really is a valid hexadecimal
** character: 0..9a..fA..F
*/
static u8 jsonHexToInt(int h){
if( !sqlite3Isxdigit(h) ) return 0;
#ifdef SQLITE_EBCDIC
h += 9*(1&~(h>>4));
#else
h += 9*(1&(h>>6));
#endif
return (u8)(h & 0xf);
}
/*
** Convert a 4-byte hex string into an integer
*/
static u32 jsonHexToInt4(const char *z){
u32 v;
v = (jsonHexToInt(z[0])<<12)
+ (jsonHexToInt(z[1])<<8)
+ (jsonHexToInt(z[2])<<4)
+ jsonHexToInt(z[3]);
v = (sqlite3HexToInt(z[0])<<12)
+ (sqlite3HexToInt(z[1])<<8)
+ (sqlite3HexToInt(z[2])<<4)
+ sqlite3HexToInt(z[3]);
return v;
}
/*
** A macro to hint to the compiler that a function should not be
** inlined.
*/
#if defined(__GNUC__)
# define JSON_NOINLINE __attribute__((noinline))
#elif defined(_MSC_VER) && _MSC_VER>=1310
# define JSON_NOINLINE __declspec(noinline)
#else
# define JSON_NOINLINE
#endif
/*
** Return true if z[] begins with 2 (or more) hexadecimal digits
*/
@@ -836,12 +963,6 @@ static const struct NanInfName {
{ 's', 'S', 4, JSONB_NULL, 4, "SNaN", "null" },
};
/*
** Magic number used for the JSON parse cache in sqlite3_get_auxdata()
*/
#define JSON_CACHE_ID (-429938) /* First cache entry */
#define JSON_CACHE_SZ 4 /* Max number of cache entries */
/*
** Compute the text of an error in JSON path syntax.
@@ -1147,8 +1268,8 @@ json_parse_restart:
if( z[j]==':' ){
j++;
}else{
if( fast_isspace(z[j]) ){
do{ j++; }while( fast_isspace(z[j]) );
if( jsonIsspace(z[j]) ){
do{ j++; }while( jsonIsspace(z[j]) );
if( z[j]==':' ){
j++;
goto parse_object_value;
@@ -1173,8 +1294,8 @@ json_parse_restart:
}else if( z[j]=='}' ){
break;
}else{
if( fast_isspace(z[j]) ){
do{ j++; }while( fast_isspace(z[j]) );
if( jsonIsspace(z[j]) ){
do{ j++; }while( jsonIsspace(z[j]) );
if( z[j]==',' ){
continue;
}else if( z[j]=='}' ){
@@ -1225,8 +1346,8 @@ json_parse_restart:
}else if( z[j]==']' ){
break;
}else{
if( fast_isspace(z[j]) ){
do{ j++; }while( fast_isspace(z[j]) );
if( jsonIsspace(z[j]) ){
do{ j++; }while( jsonIsspace(z[j]) );
if( z[j]==',' ){
continue;
}else if( z[j]==']' ){
@@ -1488,7 +1609,7 @@ json_parse_restart:
case 0x20: {
do{
i++;
}while( fast_isspace(z[i]) );
}while( jsonIsspace(z[i]) );
goto json_parse_restart;
}
case 0x0b:
@@ -1560,7 +1681,7 @@ static int jsonConvertTextToBlob(
if( pParse->oom ) i = -1;
if( i>0 ){
assert( pParse->iDepth==0 );
while( fast_isspace(zJson[i]) ) i++;
while( jsonIsspace(zJson[i]) ) i++;
if( zJson[i] ){
i += json5Whitespace(&zJson[i]);
if( zJson[i] ){
@@ -2197,7 +2318,7 @@ static void jsonReturnTextJsonFromBlob(
x.nBlob = nBlob;
jsonStringInit(&s, ctx);
jsonXlateBlobToText(&x, 0, &s);
jsonReturnString(&s);
jsonReturnString(&s, 0, 0);
}
@@ -2338,7 +2459,7 @@ static void jsonReturnFromBlob(
}else if( c=='0' ){
c = 0;
}else if( c=='x' ){
c = (jsonHexToInt(z[iIn+1])<<4) | jsonHexToInt(z[iIn+2]);
c = (sqlite3HexToInt(z[iIn+1])<<4) | sqlite3HexToInt(z[iIn+2]);
iIn += 2;
}else if( c=='\r' && z[i+1]=='\n' ){
iIn++;
@@ -2455,7 +2576,7 @@ static int jsonFunctionArgToBlob(
}
/*
** Generate a bad path error for json_extract()
** Generate a bad path error.
*/
static void jsonBadPathError(
sqlite3_context *ctx, /* The function call containing the error */
@@ -2599,6 +2720,22 @@ static JsonParse *jsonParseFuncArg(
p->nJson = sqlite3_value_bytes(pArg);
if( p->nJson==0 ) goto json_pfa_malformed;
if( p->zJson==0 ) goto json_pfa_oom;
p->aBlob = jsonCacheSearch(ctx, p->zJson, p->nJson, &p->nBlob);
if( p->aBlob ){
if( flgs & JSON_EDITABLE ){
u8 *pNew = sqlite3_malloc64( p->nBlob );
if( pNew==0 ) goto json_pfa_oom;
memcpy(pNew, p->aBlob, p->nBlob);
p->aBlob = pNew;
p->nBlobAlloc = p->nBlob;
}else{
sqlite3RCStrRef((char*)p->aBlob);
p->bBlobIsRCStr = 1;
}
return p;
}
if( flgs & JSON_KEEPERROR ) ctx = 0;
if( jsonConvertTextToBlob(p, ctx) ){
if( flgs & JSON_KEEPERROR ){
@@ -2650,7 +2787,7 @@ static void jsonReturnParse(
JsonString s;
jsonStringInit(&s, ctx);
jsonXlateBlobToText(p, 0, &s);
jsonReturnString(&s);
jsonReturnString(&s, p, ctx);
sqlite3_result_subtype(ctx, JSON_SUBTYPE);
}
}
@@ -2890,7 +3027,7 @@ static void jsonQuoteFunc(
jsonStringInit(&jx, ctx);
jsonAppendSqlValue(&jx, argv[0]);
jsonReturnString(&jx);
jsonReturnString(&jx, 0, 0);
sqlite3_result_subtype(ctx, JSON_SUBTYPE);
}
@@ -2914,7 +3051,7 @@ static void jsonArrayFunc(
jsonAppendSqlValue(&jx, argv[i]);
}
jsonAppendChar(&jx, ']');
jsonReturnString(&jx);
jsonReturnString(&jx, 0, 0);
sqlite3_result_subtype(ctx, JSON_SUBTYPE);
}
@@ -3058,7 +3195,7 @@ static void jsonExtractFunc(
if( flags & JSON_JSON ){
jsonStringInit(&jx, ctx);
jsonXlateBlobToText(p, j, &jx);
jsonReturnString(&jx);
jsonReturnString(&jx, 0, 0);
jsonStringReset(&jx);
assert( (flags & JSON_BLOB)==0 );
sqlite3_result_subtype(ctx, JSON_SUBTYPE);
@@ -3091,7 +3228,7 @@ static void jsonExtractFunc(
}
if( argc>2 ){
jsonAppendChar(&jx, ']');
jsonReturnString(&jx);
jsonReturnString(&jx, 0, 0);
if( (flags & JSON_BLOB)==0 ){
sqlite3_result_subtype(ctx, JSON_SUBTYPE);
}
@@ -3389,7 +3526,7 @@ static void jsonObjectFunc(
jsonAppendSqlValue(&jx, argv[i+1]);
}
jsonAppendChar(&jx, '}');
jsonReturnString(&jx);
jsonReturnString(&jx, 0, 0);
sqlite3_result_subtype(ctx, JSON_SUBTYPE);
}
@@ -3767,7 +3904,7 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){
jsonAppendChar(pStr, ']');
flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx));
if( pStr->eErr ){
jsonReturnString(pStr);
jsonReturnString(pStr, 0, 0);
return;
}else if( flags & JSON_BLOB ){
jsonReturnStringAsBlob(pStr);
@@ -3887,7 +4024,7 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){
pStr->pCtx = ctx;
flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx));
if( pStr->eErr ){
jsonReturnString(pStr);
jsonReturnString(pStr, 0, 0);
return;
}else if( flags & JSON_BLOB ){
jsonReturnStringAsBlob(pStr);

View File

@@ -1369,7 +1369,7 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){
/*****************************************************************************
** Reference counted string storage
** Reference counted string/blob storage
*****************************************************************************/
/*

View File

@@ -4116,6 +4116,9 @@ struct sqlite3_str {
**
** 3. Make a (read-only) copy of a read-only RCStr string using
** sqlite3RCStrRef().
**
** "String" is in the name, but an RCStr object can also be used to hold
** binary data.
*/
struct RCStr {
u64 nRCRef; /* Number of references */