From 5fa5c10784d6117df1706eb449dcd92b8396939b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Aug 2015 16:49:40 +0000 Subject: [PATCH 01/54] Begin adding an extension that provides JSON SQL functions. FossilOrigin-Name: dde8afdd8dba1d92560326dca7c1cdfedbe5e070 --- Makefile.in | 1 + Makefile.msc | 1 + ext/misc/json.c | 225 ++++++++++++++++++++++++++++++++++++++++++++++++ main.mk | 1 + manifest | 25 +++--- manifest.uuid | 2 +- src/test1.c | 2 + test/json1.test | 40 +++++++++ 8 files changed, 286 insertions(+), 11 deletions(-) create mode 100644 ext/misc/json.c create mode 100644 test/json1.test diff --git a/Makefile.in b/Makefile.in index a0f536cb23..4c83d9dfa3 100644 --- a/Makefile.in +++ b/Makefile.in @@ -417,6 +417,7 @@ TESTSRC += \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/json.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ diff --git a/Makefile.msc b/Makefile.msc index 22d3fb523e..52769889aa 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1083,6 +1083,7 @@ TESTEXT = \ $(TOP)\ext\fts5\fts5_tcl.c \ $(TOP)\ext\fts5\fts5_test_mi.c \ $(TOP)\ext\misc\ieee754.c \ + $(TOP)\ext\misc\json.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ diff --git a/ext/misc/json.c b/ext/misc/json.c new file mode 100644 index 0000000000..53442a9c6a --- /dev/null +++ b/ext/misc/json.c @@ -0,0 +1,225 @@ +/* +** 2015-08-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements JSON functions. The interface is +** modeled after MySQL JSON functions: +** +** https://dev.mysql.com/doc/refman/5.7/en/json.html +** +** JSON is pure text. JSONB is a binary encoding that is smaller and easier +** to parse but which holds the equivalent information. Conversions between +** JSON and JSONB are lossless. +** +** Most of the functions here will accept either JSON or JSONB input. The +** input is understood to be JSONB if it a BLOB and JSON if the input is +** of any other type. Functions that begin with the "json_" prefix return +** JSON and functions that begin with "jsonb_" return JSONB. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Unsigned integer types */ +typedef sqlite3_uint64 u64; +typedef unsigned int u32; +typedef unsigned char u8; + +/* An instance of this object represents a JSON string under construction. +*/ +typedef struct Json Json; +struct Json { + sqlite3_context *pCtx; /* Function context - put error messages here */ + char *zBuf; /* Append JSON text here */ + u64 nAlloc; /* Bytes of storage available in zBuf[] */ + u64 nUsed; /* Bytes of zBuf[] currently used */ + u8 bStatic; /* True if zBuf is static space */ + u8 mallocFailed; /* True if an OOM has been encountered */ + char zSpace[100]; /* Initial static space */ +}; + +/* Set the Json object to an empty string +*/ +static void jsonZero(Json *p){ + p->zBuf = p->zSpace; + p->nAlloc = sizeof(p->zSpace); + p->nUsed = 0; + p->bStatic = 1; +} + +/* Initialize the Json object +*/ +static void jsonInit(Json *p, sqlite3_context *pCtx){ + p->pCtx = pCtx; + p->mallocFailed = 0; + jsonZero(p); +} + + +/* Free all allocated memory and reset the Json object back to its +** initial state. +*/ +static void jsonReset(Json *p){ + if( !p->bStatic ) sqlite3_free(p->zBuf); + jsonZero(p); +} + + +/* Report an out-of-memory (OOM) condition +*/ +static void jsonOom(Json *p){ + p->mallocFailed = 1; + sqlite3_result_error_nomem(p->pCtx); + jsonReset(p); +} + +/* Enlarge pJson->zBuf so that it can hold at least N more bytes. +** Return zero on success. Return non-zero on an OOM error +*/ +static int jsonGrow(Json *p, u32 N){ + u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+100; + char *zNew; + if( p->bStatic ){ + if( p->mallocFailed ) return SQLITE_NOMEM; + zNew = sqlite3_malloc64(nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + memcpy(zNew, p->zBuf, p->nUsed); + p->zBuf = zNew; + p->bStatic = 0; + }else{ + zNew = sqlite3_realloc64(p->zBuf, nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + p->zBuf = zNew; + } + p->nAlloc = nTotal; + return SQLITE_OK; +} + +/* Append N bytes from zIn onto the end of the Json string. +*/ +static void jsonAppendRaw(Json *p, const char *zIn, u32 N){ + if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; +} + +/* Append the N-byte string in zIn to the end of the Json string +** under construction. Enclose the string in "..." and escape +** any double-quotes or backslash characters contained within the +** string. +*/ +static void jsonAppendString(Json *p, const char *zIn, u32 N){ + u32 i; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; + p->zBuf[p->nUsed++] = '"'; + for(i=0; inUsed+N+1-i > p->nAlloc) && jsonGrow(p,N+1-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + } + p->zBuf[p->nUsed++] = c; + } + p->zBuf[p->nUsed++] = '"'; +} + +/* Make pJson the result of the SQL function. +*/ +static void jsonResult(Json *p){ + if( p->mallocFailed==0 ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, + SQLITE_UTF8); + jsonZero(p); + } + assert( p->bStatic ); +} + +/* +** Implementation of the json_array(VALUE,...) function. Return a JSON +** array that contains all values given in arguments. Or if any argument +** is a BLOB, throw an error. +*/ +static void jsonArrayFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + Json jx; + char cSep = '['; + + jsonInit(&jx, context); + for(i=0; i Date: Wed, 12 Aug 2015 17:23:34 +0000 Subject: [PATCH 02/54] Add the json_object() function. FossilOrigin-Name: 414a95f3b79359f167858b2325fd2be223569c66 --- ext/misc/json.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++-- manifest | 17 +++++-------- manifest.uuid | 2 +- test/json1.test | 14 ++++++++++ 4 files changed, 88 insertions(+), 13 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 53442a9c6a..8acbb9bf5f 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -194,7 +194,70 @@ static void jsonArrayFunc( } } jsonAppendRaw(&jx, "]", 1); - jsonResult(&jx); + jsonResult(&jx); +} + +/* +** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON +** object that contains all name/value given in arguments. Or if any name +** is not a string or if any value is a BLOB, throw an error. +*/ +static void jsonObjectFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + Json jx; + char cSep = '{'; + const char *z; + u32 n; + + if( argc&1 ){ + sqlite3_result_error(context, "json_object() requires an even number " + "of arguments", -1); + return; + } + jsonInit(&jx, context); + for(i=0; i Date: Wed, 12 Aug 2015 19:42:08 +0000 Subject: [PATCH 03/54] Avoid reading frames that have already been checkpointed from the wal file. FossilOrigin-Name: 5669ac4a40429abc3f44540fc9d2f3b79b404bdf --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/wal.c | 12 +++++++++--- test/wal6.test | 42 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c0121c70db..ebe04a4a71 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\soptimization\sfor\sfts5\sAPI\sxInst(). -D 2015-08-12T15:36:58.855 +C Avoid\sreading\sframes\sthat\shave\salready\sbeen\scheckpointed\sfrom\sthe\swal\sfile. +D 2015-08-12T19:42:08.239 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2fc9ca6bf5949d415801c007ed3004a4bdb7c380 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -408,7 +408,7 @@ F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 082b35a25a26e3d36f365ca8cd73c1922532f05e F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 +F src/wal.c 9eb487483eed48310f999d9735263f33c71026b3 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c c745d3aa78ad1aa8982febb99f2f17ee5cbac069 @@ -1256,7 +1256,7 @@ F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test 2b5445e5da44780b9b44712f5a38523f7aeb0941 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 88b5d9a6a3d1532497ee9f4296f010d66f07e33c -F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e +F test/wal6.test 4421cd5a2fa99d29cc91ef12fb23bed171ed3a4c F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test 75c42e1bc4545c277fed212f8fc9b7723cd02216 @@ -1372,7 +1372,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f7682435278419829a46bb4cc9b5625d46549e22 -R 5c7dc1bc59e7a8c21161b1ef8857c7b8 +P efb7c9c5d0015c8c966f8d6742c05cda82fc146a +R 91c715455a10615960997cdbeb286182 +T *branch * wal-read-change +T *sym-wal-read-change * +T -sym-trunk * U dan -Z c11fd987f7eb5e43e308c10674f8c3b9 +Z 265f3376267bf1285595a2e2dc6d13aa diff --git a/manifest.uuid b/manifest.uuid index 584731300e..52606f1923 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -efb7c9c5d0015c8c966f8d6742c05cda82fc146a \ No newline at end of file +5669ac4a40429abc3f44540fc9d2f3b79b404bdf \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index f7e2594001..0e4d07d372 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2372,6 +2372,8 @@ int sqlite3WalFindFrame( u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ + u32 iFirst; + int iMinHash; /* This routine is only be called from within a read transaction. */ assert( pWal->readLock>=0 || pWal->lockError ); @@ -2382,7 +2384,10 @@ int sqlite3WalFindFrame( ** then the WAL is ignored by the reader so return early, as if the ** WAL were empty. */ - if( iLast==0 || pWal->readLock==0 ){ + if( iLast==0 + || pWal->readLock==0 + || iLast==(iFirst = walCkptInfo(pWal)->nBackfill) + ){ *piRead = 0; return SQLITE_OK; } @@ -2412,7 +2417,8 @@ int sqlite3WalFindFrame( ** This condition filters out entries that were added to the hash ** table after the current read-transaction had started. */ - for(iHash=walFramePage(iLast); iHash>=0 && iRead==0; iHash--){ + iMinHash = walFramePage(iFirst); + for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ volatile ht_slot *aHash; /* Pointer to hash table */ volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ @@ -2427,7 +2433,7 @@ int sqlite3WalFindFrame( nCollide = HASHTABLE_NSLOT; for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ + if( iFrame<=iLast && iFrame>iFirst && aPgno[aHash[iKey]]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } diff --git a/test/wal6.test b/test/wal6.test index 2498907ea9..d96166ef59 100644 --- a/test/wal6.test +++ b/test/wal6.test @@ -193,5 +193,47 @@ do_test 3.x { db2 close } {} +#------------------------------------------------------------------------- +# Check that if a wal file has been partially checkpointed, no frames are +# read from the checkpointed part. +# +reset_db +do_execsql_test 4.1 { + PRAGMA page_size = 1024; + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + PRAGMA wal_checkpoint = truncate; +} {wal 0 0 0} + +do_test 4.2 { + execsql { INSERT INTO t1 VALUES(1, 2) } + file size test.db-wal +} [wal_file_size 1 1024] + +do_test 4.3 { + sqlite3 db2 test.db + execsql { + BEGIN; + INSERT INTO t2 VALUES(3, 4); + } + execsql { PRAGMA wal_checkpoint = passive } db2 +} {0 1 1} + +do_test 4.3 { + execsql { COMMIT } + db2 close + hexio_write test.db-wal 0 [string repeat 00 2000] + sqlite3 db2 test.db +} {} + +do_test 4.4.1 { + catchsql { SELECT * FROM t1 } db2 +} {0 {1 2}} +do_test 4.4.2 { + catchsql { SELECT * FROM t2 } db2 +} {1 {database disk image is malformed}} + + finish_test From bd0621b360ec250683d77a8a5cd3b8ed49ddd017 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 13 Aug 2015 13:54:59 +0000 Subject: [PATCH 04/54] Experimental code (untested) for a JSONB datatype. FossilOrigin-Name: e3596ac7b1dd5bde3f9cae5781a6806d8d9f7166 --- ext/misc/json.c | 269 +++++++++++++++++++++++++++++++++++++++++++++++- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 273 insertions(+), 10 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 8acbb9bf5f..4f3e1c02bb 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -23,6 +23,55 @@ ** input is understood to be JSONB if it a BLOB and JSON if the input is ** of any other type. Functions that begin with the "json_" prefix return ** JSON and functions that begin with "jsonb_" return JSONB. +** +** JSONB format: +** +** A JSONB blob is a sequence of terms. Each term begins with a single +** variable length integer X which determines the type and size of the term. +** +** type = X%8 +** size = X>>3 +** +** Term types are 0 through 7 for null, true, false, integer, real, string, +** array, and object. The meaning of size depends on the type. +** +** For null, true, and false terms, the size is always 0. +** +** For integer terms, the size is the number of bytes that contains the +** integer value. The value is stored as big-endian twos-complement. +** +** For real terms, the size is always 8 and the value is a big-ending +** double-precision floating-point number. +** +** For string terms, the size is the number of bytes in the string. The +** string itself immediately follows the X integer. There are no escapes +** and the string is not zero-terminated. The string is always stored as +** UTF8. +** +** For array terms, the size is the number of bytes in content. The +** content consists of zero or more additional terms that are the elements +** of the array. +** +** For object terms, the size is the number of bytes of content. The +** content is zero or more pairs of terms. The first element of each +** pair is a string term which is the label and the second element is +** the value. +** +** Variable Length Integers: +** +** The variable length integer encoding is the 64-bit unsigned integer encoding +** originally developed for SQLite4. The encoding for each integer is between +** 1 and 9 bytes. Call those bytes A0 through A8. The encoding is as follows: +** +** If A0 is between 0 and 240 inclusive, then the value is A0. +** +** If A0 is between 241 and 248 inclusive, then the value is +** 240+256*(A0-241)+A1. +** +** If A0 is 249 then the value is 2288+256*A1+A2. +** +** If A0 is 250 or more then the value is a (A0-247)-byte big-endian +** integer taken from starting at A1. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -34,12 +83,13 @@ typedef sqlite3_uint64 u64; typedef unsigned int u32; typedef unsigned char u8; -/* An instance of this object represents a JSON string under construction. +/* An instance of this object represents a JSON string or +** JSONB blob under construction. */ typedef struct Json Json; struct Json { sqlite3_context *pCtx; /* Function context - put error messages here */ - char *zBuf; /* Append JSON text here */ + char *zBuf; /* Append JSON or JSONB content here */ u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ @@ -47,6 +97,73 @@ struct Json { char zSpace[100]; /* Initial static space */ }; +/* JSONB type values +*/ +#define JSONB_NULL 0 +#define JSONB_TRUE 1 +#define JSONB_FALSE 2 +#define JSONB_INT 3 +#define JSONB_REAL 4 +#define JSONB_STRING 5 +#define JSONB_ARRAY 6 +#define JSONB_OBJECT 7 + +#if 0 +/* +** Decode the varint in the first n bytes z[]. Write the integer value +** into *pResult and return the number of bytes in the varint. +** +** If the decode fails because there are not enough bytes in z[] then +** return 0; +*/ +static int jsonGetVarint64( + const unsigned char *z, + int n, + u64 *pResult +){ + unsigned int x; + if( n<1 ) return 0; + if( z[0]<=240 ){ + *pResult = z[0]; + return 1; + } + if( z[0]<=248 ){ + if( n<2 ) return 0; + *pResult = (z[0]-241)*256 + z[1] + 240; + return 2; + } + if( nzBuf[p->nUsed++] = '"'; } -/* Make pJson the result of the SQL function. +/* +** Write a 32-bit unsigned integer as 4 big-endian bytes. +*/ +static void jsonPutInt32(unsigned char *z, unsigned int y){ + z[0] = (unsigned char)(y>>24); + z[1] = (unsigned char)(y>>16); + z[2] = (unsigned char)(y>>8); + z[3] = (unsigned char)(y); +} + + +/* Write integer X as a variable-length integer into the buffer z[]. +** z[] is guaranteed to be at least 9 bytes in length. Return the +** number of bytes written. +*/ +int jsonPutVarint64(char *zIn, u64 x){ + unsigned char *z = (unsigned char*)zIn; + unsigned int w, y; + if( x<=240 ){ + z[0] = (unsigned char)x; + return 1; + } + if( x<=2287 ){ + y = (unsigned int)(x - 240); + z[0] = (unsigned char)(y/256 + 241); + z[1] = (unsigned char)(y%256); + return 2; + } + if( x<=67823 ){ + y = (unsigned int)(x - 2288); + z[0] = 249; + z[1] = (unsigned char)(y/256); + z[2] = (unsigned char)(y%256); + return 3; + } + y = (unsigned int)x; + w = (unsigned int)(x>>32); + if( w==0 ){ + if( y<=16777215 ){ + z[0] = 250; + z[1] = (unsigned char)(y>>16); + z[2] = (unsigned char)(y>>8); + z[3] = (unsigned char)(y); + return 4; + } + z[0] = 251; + jsonPutInt32(z+1, y); + return 5; + } + if( w<=255 ){ + z[0] = 252; + z[1] = (unsigned char)w; + jsonPutInt32(z+2, y); + return 6; + } + if( w<=65535 ){ + z[0] = 253; + z[1] = (unsigned char)(w>>8); + z[2] = (unsigned char)w; + jsonPutInt32(z+3, y); + return 7; + } + if( w<=16777215 ){ + z[0] = 254; + z[1] = (unsigned char)(w>>16); + z[2] = (unsigned char)(w>>8); + z[3] = (unsigned char)w; + jsonPutInt32(z+4, y); + return 8; + } + z[0] = 255; + jsonPutInt32(z+1, w); + jsonPutInt32(z+5, y); + return 9; +} + + +/* Append integer X as a variable-length integer on the JSONB currently +** under construction in p. +*/ +static void jsonAppendVarint(Json *p, u64 X){ + if( (p->nUsed+9 > p->nAlloc) && jsonGrow(p,9)!=0 ) return; + p->nUsed += jsonPutVarint64(p->zBuf+p->nUsed, X); +} + +/* Make the JSON in p the result of the SQL function. */ static void jsonResult(Json *p){ if( p->mallocFailed==0 ){ @@ -150,6 +352,17 @@ static void jsonResult(Json *p){ assert( p->bStatic ); } +/* Make the JSONB in p the result of the SQL function. +*/ +static void jsonbResult(Json *p){ + if( p->mallocFailed==0 ){ + sqlite3_result_blob(p->pCtx, p->zBuf, p->nUsed, + p->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + jsonZero(p); + } + assert( p->bStatic ); +} + /* ** Implementation of the json_array(VALUE,...) function. Return a JSON ** array that contains all values given in arguments. Or if any argument @@ -197,6 +410,55 @@ static void jsonArrayFunc( jsonResult(&jx); } +/* +** Implementation of the jsonb_array(VALUE,...) function. Return a JSON +** array that contains all values given in arguments. Or if any argument +** is a BLOB, throw an error. +*/ +static void jsonbArrayFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + Json jx; + + jsonInit(&jx, context); + jx.nUsed = 5; + for(i=0; i Date: Thu, 13 Aug 2015 20:23:46 +0000 Subject: [PATCH 05/54] When searching the wal file for a frame, do not search that part that was already checkpointed when the snapshot being read was at the head of the wal file. FossilOrigin-Name: 90760e72327eb0593cbfa6d7058b554198cd8044 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/wal.c | 36 ++++++++++++++++++++++++------------ 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index ebe04a4a71..e20dbce225 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sreading\sframes\sthat\shave\salready\sbeen\scheckpointed\sfrom\sthe\swal\sfile. -D 2015-08-12T19:42:08.239 +C When\ssearching\sthe\swal\sfile\sfor\sa\sframe,\sdo\snot\ssearch\sthat\spart\sthat\swas\salready\scheckpointed\swhen\sthe\ssnapshot\sbeing\sread\swas\sat\sthe\shead\sof\sthe\swal\sfile. +D 2015-08-13T20:23:46.835 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2fc9ca6bf5949d415801c007ed3004a4bdb7c380 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -408,7 +408,7 @@ F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 082b35a25a26e3d36f365ca8cd73c1922532f05e F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 9eb487483eed48310f999d9735263f33c71026b3 +F src/wal.c 8cd07f1f99e1a81346db1c9da879bef6c6f97cf6 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c c745d3aa78ad1aa8982febb99f2f17ee5cbac069 @@ -1372,10 +1372,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P efb7c9c5d0015c8c966f8d6742c05cda82fc146a -R 91c715455a10615960997cdbeb286182 -T *branch * wal-read-change -T *sym-wal-read-change * -T -sym-trunk * +P 5669ac4a40429abc3f44540fc9d2f3b79b404bdf +R 0be4044b3773f95bd61300112256d197 U dan -Z 265f3376267bf1285595a2e2dc6d13aa +Z f0c98bb462164518b46029fee8374b58 diff --git a/manifest.uuid b/manifest.uuid index 52606f1923..84745b0a24 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5669ac4a40429abc3f44540fc9d2f3b79b404bdf \ No newline at end of file +90760e72327eb0593cbfa6d7058b554198cd8044 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 0e4d07d372..cf8f1d4e66 100644 --- a/src/wal.c +++ b/src/wal.c @@ -428,6 +428,7 @@ struct Wal { u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ + u32 minFrame; /* Ignore wal frames before this one */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG @@ -2296,12 +2297,27 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry ** instead. ** - ** This does not guarantee that the copy of the wal-index header is up to - ** date before proceeding. That would not be possible without somehow - ** blocking writers. It only guarantees that a dangerous checkpoint or - ** log-wrap (either of which would require an exclusive lock on - ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. + ** Before checking that the live wal-index header has not changed + ** since it was read, set Wal.minFrame to the first frame in the wal + ** file that has not yet been checkpointed. This client will not need + ** to read any frames earlier than minFrame from the wal file - they + ** can be safely read directly from the database file. + ** + ** Because a ShmBarrier() call is made between taking the copy of + ** nBackfill and checking that the wal-header in shared-memory still + ** matches the one cached in pWal->hdr, it is guaranteed that the + ** checkpointer that set nBackfill was not working with a wal-index + ** header newer than that cached in pWal->hdr. If it were, that could + ** cause a problem. The checkpointer could omit to checkpoint + ** a version of page X that lies before pWal->minFrame (call that version + ** A) on the basis that there is a newer version (version B) of the same + ** page later in the wal file. But if version B happens to like past + ** frame pWal->hdr.mxFrame - then the client would incorrectly assume + ** that it can read version A from the database file. However, since + ** we can guarantee that the checkpointer that set nBackfill could not + ** see any pages past pWal->hdr.mxFrame, this problem does not come up. */ + pWal->minFrame = pInfo->nBackfill+1; walShmBarrier(pWal); if( pInfo->aReadMark[mxI]!=mxReadMark || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) @@ -2372,7 +2388,6 @@ int sqlite3WalFindFrame( u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ - u32 iFirst; int iMinHash; /* This routine is only be called from within a read transaction. */ @@ -2384,10 +2399,7 @@ int sqlite3WalFindFrame( ** then the WAL is ignored by the reader so return early, as if the ** WAL were empty. */ - if( iLast==0 - || pWal->readLock==0 - || iLast==(iFirst = walCkptInfo(pWal)->nBackfill) - ){ + if( iLast==0 || pWal->readLock==0 ){ *piRead = 0; return SQLITE_OK; } @@ -2417,7 +2429,7 @@ int sqlite3WalFindFrame( ** This condition filters out entries that were added to the hash ** table after the current read-transaction had started. */ - iMinHash = walFramePage(iFirst); + iMinHash = walFramePage(pWal->minFrame); for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ volatile ht_slot *aHash; /* Pointer to hash table */ volatile u32 *aPgno; /* Pointer to array of page numbers */ @@ -2433,7 +2445,7 @@ int sqlite3WalFindFrame( nCollide = HASHTABLE_NSLOT; for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && iFrame>iFirst && aPgno[aHash[iKey]]==pgno ){ + if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } From e9c37f32e57ba56b4ba14b5445a65ebaecd641e2 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 15 Aug 2015 21:25:36 +0000 Subject: [PATCH 06/54] Add a prototype JSON parser. FossilOrigin-Name: 789ba487000aa73621a41d115ad5de455ea8ea31 --- ext/misc/json.c | 302 +++++++++++++++++++++++++++++++++++++++++++++--- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 291 insertions(+), 25 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 4f3e1c02bb..2fce79610c 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -77,6 +77,7 @@ SQLITE_EXTENSION_INIT1 #include #include +#include /* Unsigned integer types */ typedef sqlite3_uint64 u64; @@ -93,20 +94,41 @@ struct Json { u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ - u8 mallocFailed; /* True if an OOM has been encountered */ + u8 oom; /* True if an OOM has been encountered */ char zSpace[100]; /* Initial static space */ }; -/* JSONB type values +/* JSON type values */ -#define JSONB_NULL 0 -#define JSONB_TRUE 1 -#define JSONB_FALSE 2 -#define JSONB_INT 3 -#define JSONB_REAL 4 -#define JSONB_STRING 5 -#define JSONB_ARRAY 6 -#define JSONB_OBJECT 7 +#define JSON_NULL 0 +#define JSON_TRUE 1 +#define JSON_FALSE 2 +#define JSON_INT 3 +#define JSON_REAL 4 +#define JSON_STRING 5 +#define JSON_ARRAY 6 +#define JSON_OBJECT 7 + +/* A single node of parsed JSON +*/ +typedef struct JsonNode JsonNode; +struct JsonNode { + u32 eType; /* One of the JSON_ type values */ + u32 n; /* Bytes of content, or number of sub-nodes */ + const char *zContent; /* Content for JSON_INT, JSON_REAL, or JSON_STRING */ +}; + +/* A completely parsed JSON string +*/ +typedef struct JsonParse JsonParse; +struct JsonParse { + u32 nNode; /* Number of slots of aNode[] used */ + u32 nAlloc; /* Number of slots of aNode[] allocated */ + JsonNode *aNode; /* Array of nodes containing the parse */ + const char *zJson; /* Original JSON string */ + u8 oom; /* Set to true if out of memory */ +}; + #if 0 /* @@ -177,7 +199,7 @@ static void jsonZero(Json *p){ */ static void jsonInit(Json *p, sqlite3_context *pCtx){ p->pCtx = pCtx; - p->mallocFailed = 0; + p->oom = 0; jsonZero(p); } @@ -194,7 +216,7 @@ static void jsonReset(Json *p){ /* Report an out-of-memory (OOM) condition */ static void jsonOom(Json *p){ - p->mallocFailed = 1; + p->oom = 1; sqlite3_result_error_nomem(p->pCtx); jsonReset(p); } @@ -206,7 +228,7 @@ static int jsonGrow(Json *p, u32 N){ u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+100; char *zNew; if( p->bStatic ){ - if( p->mallocFailed ) return SQLITE_NOMEM; + if( p->oom ) return SQLITE_NOMEM; zNew = sqlite3_malloc64(nTotal); if( zNew==0 ){ jsonOom(p); @@ -235,6 +257,12 @@ static void jsonAppendRaw(Json *p, const char *zIn, u32 N){ p->nUsed += N; } +/* Append the zero-terminated string zIn +*/ +static void jsonAppend(Json *p, const char *zIn){ + jsonAppendRaw(p, zIn, (u32)strlen(zIn)); +} + /* Append the N-byte string in zIn to the end of the Json string ** under construction. Enclose the string in "..." and escape ** any double-quotes or backslash characters contained within the @@ -343,7 +371,7 @@ static void jsonAppendVarint(Json *p, u64 X){ /* Make the JSON in p the result of the SQL function. */ static void jsonResult(Json *p){ - if( p->mallocFailed==0 ){ + if( p->oom==0 ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, SQLITE_UTF8); @@ -355,7 +383,7 @@ static void jsonResult(Json *p){ /* Make the JSONB in p the result of the SQL function. */ static void jsonbResult(Json *p){ - if( p->mallocFailed==0 ){ + if( p->oom==0 ){ sqlite3_result_blob(p->pCtx, p->zBuf, p->nUsed, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free); jsonZero(p); @@ -428,7 +456,7 @@ static void jsonbArrayFunc( for(i=0; iaNode[] of the +** new node, or -1 if a memory allocation fails. +*/ +static int jsonParseAddNode( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + JsonNode *p; + if( pParse->nNode>=pParse->nAlloc ){ + u32 nNew; + JsonNode *pNew; + if( pParse->oom ) return -1; + nNew = pParse->nAlloc*2 + 10; + if( nNew<=pParse->nNode ){ + pParse->oom = 1; + return -1; + } + pNew = sqlite3_realloc(pParse->aNode, sizeof(JsonNode)*nNew); + if( pNew==0 ){ + pParse->oom = 1; + return -1; + } + pParse->nAlloc = nNew; + pParse->aNode = pNew; + } + p = &pParse->aNode[pParse->nNode]; + p->eType = eType; + p->n = n; + p->zContent = zContent; + return pParse->nNode++; +} + +/* +** Parse a single JSON value which begins at pParse->zJson[i]. Return the +** index of the first character past the end of the value parsed. +** +** Return negative for a syntax error. Special cases: return -2 if the +** first non-whitespace character is '}' and return -3 if the first +** non-whitespace character is ']'. +*/ +static int jsonParseValue(JsonParse *pParse, u32 i){ + char c; + u32 j; + u32 iThis; + int x; + while( isspace(pParse->zJson[i]) ){ i++; } + if( (c = pParse->zJson[i])==0 ) return 0; + if( c=='{' ){ + /* Parse object */ + iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( isspace(pParse->zJson[j]) ){ j++; } + x = jsonParseValue(pParse, j); + if( x<0 ){ + if( x==(-2) && pParse->nNode==iThis+1 ) return j+1; + return -1; + } + if( pParse->aNode[pParse->nNode-1].eType!=JSON_STRING ) return -1; + j = x; + while( isspace(pParse->zJson[j]) ){ j++; } + if( pParse->zJson[j]!=':' ) return -1; + j++; + x = jsonParseValue(pParse, j); + if( x<0 ) return -1; + j = x; + while( isspace(pParse->zJson[j]) ){ j++; } + c = pParse->zJson[j]; + if( c==',' ) continue; + if( c!='}' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - iThis - 1; + return j+1; + }else if( c=='[' ){ + /* Parse array */ + iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( isspace(pParse->zJson[j]) ){ j++; } + x = jsonParseValue(pParse, j); + if( x<0 ){ + if( x==(-3) && pParse->nNode==iThis+1 ) return j+1; + return -1; + } + j = x; + while( isspace(pParse->zJson[j]) ){ j++; } + c = pParse->zJson[j]; + if( c==',' ) continue; + if( c!=']' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - iThis - 1; + return j+1; + }else if( c=='"' ){ + /* Parse string */ + j = i+1; + for(;;){ + c = pParse->zJson[j]; + if( c==0 ) return -1; + if( c=='\\' ){ + c = pParse->zJson[++j]; + if( c==0 ) return -1; + }else if( c=='"' ){ + break; + } + j++; + } + jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]); + return j+1; + }else if( c=='n' + && strncmp(pParse->zJson+i,"null",4)==0 + && !isalnum(pParse->zJson[i+5]) ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return i+4; + }else if( c=='t' + && strncmp(pParse->zJson+i,"true",4)==0 + && !isalnum(pParse->zJson[i+5]) ){ + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + return i+4; + }else if( c=='f' + && strncmp(pParse->zJson+i,"false",5)==0 + && !isalnum(pParse->zJson[i+6]) ){ + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + return i+5; + }else if( c=='-' || (c>='0' && c<='9') ){ + /* Parse number */ + u8 seenDP = 0; + u8 seenE = 0; + j = i+1; + for(;; j++){ + c = pParse->zJson[j]; + if( c>='0' && c<='9' ) continue; + if( c=='.' ){ + if( pParse->zJson[j-1]=='-' ) return -1; + if( seenDP ) return -1; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( pParse->zJson[j-1]<'0' ) return -1; + if( seenE ) return -1; + seenDP = seenE = 1; + c = pParse->zJson[j+1]; + if( c=='+' || c=='-' ) j++; + continue; + } + break; + } + if( pParse->zJson[j-1]<'0' ) return -1; + jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, + j - i, &pParse->zJson[i]); + return j; + }else if( c=='}' ){ + return -2; /* End of {...} */ + }else if( c==']' ){ + return -3; /* End of [...] */ + }else{ + return -1; /* Syntax error */ + } +} + +/* +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory associated with +** pParse. +** +** pParse is uninitialized when this routine is called. +*/ +static int jsonParse(JsonParse *pParse, const char *zJson){ + int i; + if( zJson==0 ) return 1; + memset(pParse, 0, sizeof(*pParse)); + pParse->zJson = zJson; + i = jsonParseValue(pParse, 0); + if( i>0 ){ + while( isspace(zJson[i]) ) i++; + if( zJson[i] ) i = -1; + } + if( i<0 ){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + pParse->nNode = 0; + pParse->nAlloc = 0; + return 1; + } + return 0; +} + +/* +** The json_debug(JSON) function returns a string which describes +** a parse of the JSON provided. Or it returns NULL if JSON is not +** well-formed. +*/ +static void jsonDebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Json s; /* Output string - not real JSON */ + JsonParse x; /* The parse */ + u32 i; + char zBuf[50]; + static const char *azType[] = { + "NULL", "TRUE", "FALSE", "INT", "REAL", "STRING", "ARRAY", "OBJECT" + }; + + assert( argc==1 ); + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonInit(&s, context); + for(i=0; i=JSON_INT ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, " n: %u\n", x.aNode[i].n); + jsonAppend(&s, zBuf); + } + if( x.aNode[i].zContent!=0 ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, " ofst: %u\n", + (u32)(x.aNode[i].zContent - x.zJson)); + jsonAppend(&s, zBuf); + jsonAppendRaw(&s, " text: ", 8); + jsonAppendRaw(&s, x.aNode[i].zContent, x.aNode[i].n); + jsonAppendRaw(&s, "\n", 1); + } + } + sqlite3_free(x.aNode); + jsonResult(&s); +} + #ifdef _WIN32 __declspec(dllexport) #endif @@ -540,6 +805,7 @@ int sqlite3_json_init( { "json_array", -1, jsonArrayFunc }, { "jsonb_array", -1, jsonbArrayFunc }, { "json_object", -1, jsonObjectFunc }, + { "json_debug", 1, jsonDebugFunc }, }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ diff --git a/manifest b/manifest index 1d41c76aaf..acc6cbf136 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\scode\s(untested)\sfor\sa\sJSONB\sdatatype. -D 2015-08-13T13:54:59.587 +C Add\sa\sprototype\sJSON\sparser. +D 2015-08-15T21:25:37.000 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 748c5bffa8840f1b0bdc4b5ff672949c31c7f436 +F ext/misc/json.c f4fa58cb8c6a7e13ae9a357e8c5783f43c60f595 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 414a95f3b79359f167858b2325fd2be223569c66 -R 45e5df2bce81d8e555551bffb25d0cb6 +P e3596ac7b1dd5bde3f9cae5781a6806d8d9f7166 +R 45c6bd1bef56b998d07e7b3125f794b3 U drh -Z 62911abad9701038f8848b5004ef4c38 +Z e2e8ec51bb6080feb16e3eda35039c7d diff --git a/manifest.uuid b/manifest.uuid index 58f3872eaf..7cd175e6dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3596ac7b1dd5bde3f9cae5781a6806d8d9f7166 \ No newline at end of file +789ba487000aa73621a41d115ad5de455ea8ea31 \ No newline at end of file From b2cd10eaa920e143df093432caa0ce97859a4839 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 15 Aug 2015 21:29:14 +0000 Subject: [PATCH 07/54] Fix off-by-one error when parsing primitive JSON types "true", "false", and "null". FossilOrigin-Name: 42c15c1e36b5077646fef99028cf12e587a45023 --- ext/misc/json.c | 6 +++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 2fce79610c..f81cbf75a7 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -666,17 +666,17 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ return j+1; }else if( c=='n' && strncmp(pParse->zJson+i,"null",4)==0 - && !isalnum(pParse->zJson[i+5]) ){ + && !isalnum(pParse->zJson[i+4]) ){ jsonParseAddNode(pParse, JSON_NULL, 0, 0); return i+4; }else if( c=='t' && strncmp(pParse->zJson+i,"true",4)==0 - && !isalnum(pParse->zJson[i+5]) ){ + && !isalnum(pParse->zJson[i+4]) ){ jsonParseAddNode(pParse, JSON_TRUE, 0, 0); return i+4; }else if( c=='f' && strncmp(pParse->zJson+i,"false",5)==0 - && !isalnum(pParse->zJson[i+6]) ){ + && !isalnum(pParse->zJson[i+5]) ){ jsonParseAddNode(pParse, JSON_FALSE, 0, 0); return i+5; }else if( c=='-' || (c>='0' && c<='9') ){ diff --git a/manifest b/manifest index acc6cbf136..40126ac50b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sprototype\sJSON\sparser. -D 2015-08-15T21:25:37.000 +C Fix\soff-by-one\serror\swhen\sparsing\sprimitive\sJSON\stypes\s"true",\s"false",\sand\n"null". +D 2015-08-15T21:29:14.831 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c f4fa58cb8c6a7e13ae9a357e8c5783f43c60f595 +F ext/misc/json.c 4f2ea491dd4f24d7a6be299af71093e6008b2f75 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e3596ac7b1dd5bde3f9cae5781a6806d8d9f7166 -R 45c6bd1bef56b998d07e7b3125f794b3 +P 789ba487000aa73621a41d115ad5de455ea8ea31 +R 11d4a720790fbd16eb396e67acd5adf6 U drh -Z e2e8ec51bb6080feb16e3eda35039c7d +Z 4f36882f0704e134b10a4112fe9bc870 diff --git a/manifest.uuid b/manifest.uuid index 7cd175e6dd..76fe7c8b07 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -789ba487000aa73621a41d115ad5de455ea8ea31 \ No newline at end of file +42c15c1e36b5077646fef99028cf12e587a45023 \ No newline at end of file From 5634cc05e40abcefd891c641a48e4cafea61b128 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Aug 2015 11:28:03 +0000 Subject: [PATCH 08/54] Abandon the JSONB format for now. (We may return to it in the future.) Add a function to render a JSON parse. FossilOrigin-Name: 9703c0aa18ae43375af876474b818e504e1c10a5 --- ext/misc/json.c | 428 ++++++++++++++++++------------------------------ manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 166 insertions(+), 276 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index f81cbf75a7..2e37835708 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -15,63 +15,9 @@ ** ** https://dev.mysql.com/doc/refman/5.7/en/json.html ** -** JSON is pure text. JSONB is a binary encoding that is smaller and easier -** to parse but which holds the equivalent information. Conversions between -** JSON and JSONB are lossless. -** -** Most of the functions here will accept either JSON or JSONB input. The -** input is understood to be JSONB if it a BLOB and JSON if the input is -** of any other type. Functions that begin with the "json_" prefix return -** JSON and functions that begin with "jsonb_" return JSONB. -** -** JSONB format: -** -** A JSONB blob is a sequence of terms. Each term begins with a single -** variable length integer X which determines the type and size of the term. -** -** type = X%8 -** size = X>>3 -** -** Term types are 0 through 7 for null, true, false, integer, real, string, -** array, and object. The meaning of size depends on the type. -** -** For null, true, and false terms, the size is always 0. -** -** For integer terms, the size is the number of bytes that contains the -** integer value. The value is stored as big-endian twos-complement. -** -** For real terms, the size is always 8 and the value is a big-ending -** double-precision floating-point number. -** -** For string terms, the size is the number of bytes in the string. The -** string itself immediately follows the X integer. There are no escapes -** and the string is not zero-terminated. The string is always stored as -** UTF8. -** -** For array terms, the size is the number of bytes in content. The -** content consists of zero or more additional terms that are the elements -** of the array. -** -** For object terms, the size is the number of bytes of content. The -** content is zero or more pairs of terms. The first element of each -** pair is a string term which is the label and the second element is -** the value. -** -** Variable Length Integers: -** -** The variable length integer encoding is the 64-bit unsigned integer encoding -** originally developed for SQLite4. The encoding for each integer is between -** 1 and 9 bytes. Call those bytes A0 through A8. The encoding is as follows: -** -** If A0 is between 0 and 240 inclusive, then the value is A0. -** -** If A0 is between 241 and 248 inclusive, then the value is -** 240+256*(A0-241)+A1. -** -** If A0 is 249 then the value is 2288+256*A1+A2. -** -** If A0 is 250 or more then the value is a (A0-247)-byte big-endian -** integer taken from starting at A1. +** For the time being, all JSON is stored as pure text. (We might add +** a JSONB type in the future which stores a binary encoding of JSON in +** a BLOB, but there is no support for JSONB in the current implementation.) */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -84,13 +30,14 @@ typedef sqlite3_uint64 u64; typedef unsigned int u32; typedef unsigned char u8; -/* An instance of this object represents a JSON string or -** JSONB blob under construction. +/* An instance of this object represents a JSON string +** under construction. Really, this is a generic string accumulator +** that can be and is used to create strings other than JSON. */ typedef struct Json Json; struct Json { sqlite3_context *pCtx; /* Function context - put error messages here */ - char *zBuf; /* Append JSON or JSONB content here */ + char *zBuf; /* Append JSON content here */ u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ @@ -113,9 +60,10 @@ struct Json { */ typedef struct JsonNode JsonNode; struct JsonNode { - u32 eType; /* One of the JSON_ type values */ + u8 eType; /* One of the JSON_ type values */ + u8 bRaw; /* Content is raw, rather than JSON encoded */ u32 n; /* Bytes of content, or number of sub-nodes */ - const char *zContent; /* Content for JSON_INT, JSON_REAL, or JSON_STRING */ + const char *zJContent; /* JSON content */ }; /* A completely parsed JSON string @@ -129,63 +77,6 @@ struct JsonParse { u8 oom; /* Set to true if out of memory */ }; - -#if 0 -/* -** Decode the varint in the first n bytes z[]. Write the integer value -** into *pResult and return the number of bytes in the varint. -** -** If the decode fails because there are not enough bytes in z[] then -** return 0; -*/ -static int jsonGetVarint64( - const unsigned char *z, - int n, - u64 *pResult -){ - unsigned int x; - if( n<1 ) return 0; - if( z[0]<=240 ){ - *pResult = z[0]; - return 1; - } - if( z[0]<=248 ){ - if( n<2 ) return 0; - *pResult = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( nnUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; + p->zBuf[p->nUsed++] = c; +} + /* Append the N-byte string in zIn to the end of the Json string ** under construction. Enclose the string in "..." and escape ** any double-quotes or backslash characters contained within the @@ -283,91 +181,6 @@ static void jsonAppendString(Json *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = '"'; } -/* -** Write a 32-bit unsigned integer as 4 big-endian bytes. -*/ -static void jsonPutInt32(unsigned char *z, unsigned int y){ - z[0] = (unsigned char)(y>>24); - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); -} - - -/* Write integer X as a variable-length integer into the buffer z[]. -** z[] is guaranteed to be at least 9 bytes in length. Return the -** number of bytes written. -*/ -int jsonPutVarint64(char *zIn, u64 x){ - unsigned char *z = (unsigned char*)zIn; - unsigned int w, y; - if( x<=240 ){ - z[0] = (unsigned char)x; - return 1; - } - if( x<=2287 ){ - y = (unsigned int)(x - 240); - z[0] = (unsigned char)(y/256 + 241); - z[1] = (unsigned char)(y%256); - return 2; - } - if( x<=67823 ){ - y = (unsigned int)(x - 2288); - z[0] = 249; - z[1] = (unsigned char)(y/256); - z[2] = (unsigned char)(y%256); - return 3; - } - y = (unsigned int)x; - w = (unsigned int)(x>>32); - if( w==0 ){ - if( y<=16777215 ){ - z[0] = 250; - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); - return 4; - } - z[0] = 251; - jsonPutInt32(z+1, y); - return 5; - } - if( w<=255 ){ - z[0] = 252; - z[1] = (unsigned char)w; - jsonPutInt32(z+2, y); - return 6; - } - if( w<=65535 ){ - z[0] = 253; - z[1] = (unsigned char)(w>>8); - z[2] = (unsigned char)w; - jsonPutInt32(z+3, y); - return 7; - } - if( w<=16777215 ){ - z[0] = 254; - z[1] = (unsigned char)(w>>16); - z[2] = (unsigned char)(w>>8); - z[3] = (unsigned char)w; - jsonPutInt32(z+4, y); - return 8; - } - z[0] = 255; - jsonPutInt32(z+1, w); - jsonPutInt32(z+5, y); - return 9; -} - - -/* Append integer X as a variable-length integer on the JSONB currently -** under construction in p. -*/ -static void jsonAppendVarint(Json *p, u64 X){ - if( (p->nUsed+9 > p->nAlloc) && jsonGrow(p,9)!=0 ) return; - p->nUsed += jsonPutVarint64(p->zBuf+p->nUsed, X); -} - /* Make the JSON in p the result of the SQL function. */ static void jsonResult(Json *p){ @@ -380,15 +193,109 @@ static void jsonResult(Json *p){ assert( p->bStatic ); } -/* Make the JSONB in p the result of the SQL function. +/* +** Convert the JsonNode pNode into a pure JSON string and +** append to pOut. Subsubstructure is also included. Return +** the number of JsonNode objects that are encoded. */ -static void jsonbResult(Json *p){ - if( p->oom==0 ){ - sqlite3_result_blob(p->pCtx, p->zBuf, p->nUsed, - p->bStatic ? SQLITE_TRANSIENT : sqlite3_free); - jsonZero(p); +static int jsonRenderNode(JsonNode *pNode, Json *pOut){ + u32 j = 0; + switch( pNode->eType ){ + case JSON_NULL: { + jsonAppendRaw(pOut, "null", 4); + break; + } + case JSON_TRUE: { + jsonAppendRaw(pOut, "true", 4); + break; + } + case JSON_FALSE: { + jsonAppendRaw(pOut, "false", 5); + break; + } + case JSON_STRING: { + if( pNode->bRaw ){ + jsonAppendString(pOut, pNode->zJContent, pNode->n); + break; + } + /* Fall through into the next case */ + } + case JSON_REAL: + case JSON_INT: { + jsonAppendRaw(pOut, pNode->zJContent, pNode->n); + break; + } + case JSON_ARRAY: { + jsonAppendChar(pOut, '['); + j = 0; + while( jn ){ + if( j>0 ) jsonAppendChar(pOut, ','); + j += jsonRenderNode(&pNode[j+1], pOut); + } + jsonAppendChar(pOut, ']'); + break; + } + case JSON_OBJECT: { + jsonAppendChar(pOut, '{'); + j = 0; + while( jn ){ + if( j>0 ) jsonAppendChar(pOut, ','); + j += jsonRenderNode(&pNode[j+1], pOut); + jsonAppendChar(pOut, ':'); + j += jsonRenderNode(&pNode[j+1], pOut); + } + jsonAppendChar(pOut, '}'); + break; + } + } + return j+1; +} + +/* +** Make the JsonNode the return value of the function. +*/ +static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ + switch( pNode->eType ){ + case JSON_NULL: { + sqlite3_result_null(pCtx); + break; + } + case JSON_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSON_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + + /* FIXME: We really want to do text->numeric conversion on these. + ** Doing so would be easy if these were internal routines, but the + ** necessary interfaces are not exposed for doing it as a loadable + ** extension. */ + case JSON_REAL: + case JSON_INT: { + sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); + break; + } + + case JSON_STRING: { + if( pNode->bRaw ){ + sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); + }else{ + /* Translate JSON formatted string into raw text */ + } + break; + } + case JSON_ARRAY: + case JSON_OBJECT: { + Json s; + jsonInit(&s, pCtx); + jsonRenderNode(pNode, &s); + jsonResult(&s); + break; + } } - assert( p->bStatic ); } /* @@ -438,55 +345,6 @@ static void jsonArrayFunc( jsonResult(&jx); } -/* -** Implementation of the jsonb_array(VALUE,...) function. Return a JSON -** array that contains all values given in arguments. Or if any argument -** is a BLOB, throw an error. -*/ -static void jsonbArrayFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int i; - Json jx; - - jsonInit(&jx, context); - jx.nUsed = 5; - for(i=0; iaNode = pNew; } p = &pParse->aNode[pParse->nNode]; - p->eType = eType; + p->eType = (u8)eType; + p->bRaw = 0; p->n = n; - p->zContent = zContent; + p->zJContent = zContent; return pParse->nNode++; } @@ -744,11 +603,11 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ } /* -** The json_debug(JSON) function returns a string which describes +** The json_parse(JSON) function returns a string which describes ** a parse of the JSON provided. Or it returns NULL if JSON is not ** well-formed. */ -static void jsonDebugFunc( +static void jsonParseFunc( sqlite3_context *context, int argc, sqlite3_value **argv @@ -774,12 +633,12 @@ static void jsonDebugFunc( sqlite3_snprintf(sizeof(zBuf), zBuf, " n: %u\n", x.aNode[i].n); jsonAppend(&s, zBuf); } - if( x.aNode[i].zContent!=0 ){ + if( x.aNode[i].zJContent!=0 ){ sqlite3_snprintf(sizeof(zBuf), zBuf, " ofst: %u\n", - (u32)(x.aNode[i].zContent - x.zJson)); + (u32)(x.aNode[i].zJContent - x.zJson)); jsonAppend(&s, zBuf); jsonAppendRaw(&s, " text: ", 8); - jsonAppendRaw(&s, x.aNode[i].zContent, x.aNode[i].n); + jsonAppendRaw(&s, x.aNode[i].zJContent, x.aNode[i].n); jsonAppendRaw(&s, "\n", 1); } } @@ -787,6 +646,36 @@ static void jsonDebugFunc( jsonResult(&s); } +/* +** The json_test1(JSON) function parses and rebuilds the JSON string. +*/ +static void jsonTest1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonReturn(x.aNode, context); + sqlite3_free(x.aNode); +} + +/* +** The json_nodecount(JSON) function returns the number of nodes in the +** input JSON string. +*/ +static void jsonNodeCountFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + sqlite3_result_int64(context, x.nNode); + sqlite3_free(x.aNode); +} + + #ifdef _WIN32 __declspec(dllexport) #endif @@ -803,9 +692,10 @@ int sqlite3_json_init( void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { { "json_array", -1, jsonArrayFunc }, - { "jsonb_array", -1, jsonbArrayFunc }, { "json_object", -1, jsonObjectFunc }, - { "json_debug", 1, jsonDebugFunc }, + { "json_parse", 1, jsonParseFunc }, /* DEBUG */ + { "json_test1", 1, jsonTest1Func }, /* DEBUG */ + { "json_nodecount", 1, jsonNodeCountFunc }, /* DEBUG */ }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ diff --git a/manifest b/manifest index 40126ac50b..a89a28af9e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\soff-by-one\serror\swhen\sparsing\sprimitive\sJSON\stypes\s"true",\s"false",\sand\n"null". -D 2015-08-15T21:29:14.831 +C Abandon\sthe\sJSONB\sformat\sfor\snow.\s\s(We\smay\sreturn\sto\sit\sin\sthe\sfuture.)\s\sAdd\na\sfunction\sto\srender\sa\sJSON\sparse. +D 2015-08-17T11:28:03.070 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 4f2ea491dd4f24d7a6be299af71093e6008b2f75 +F ext/misc/json.c fce2fee3ac62dd53cc502cf2673b2ee5947d702d F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 789ba487000aa73621a41d115ad5de455ea8ea31 -R 11d4a720790fbd16eb396e67acd5adf6 +P 42c15c1e36b5077646fef99028cf12e587a45023 +R 4717cf337d2cdc65a44f3f6bb72a9c97 U drh -Z 4f36882f0704e134b10a4112fe9bc870 +Z 1ecceeb77cb551a38df879c773148819 diff --git a/manifest.uuid b/manifest.uuid index 76fe7c8b07..669996d2c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42c15c1e36b5077646fef99028cf12e587a45023 \ No newline at end of file +9703c0aa18ae43375af876474b818e504e1c10a5 \ No newline at end of file From 987eb1fa9b4a26ea00bf105c9725b20d27dbde51 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Aug 2015 15:17:37 +0000 Subject: [PATCH 09/54] Initial implementation for json_array_length(), json_extract(), and json_type(). FossilOrigin-Name: 39983204515837e7bd574cf47918e493acc03d1f --- ext/misc/json.c | 501 +++++++++++++++++++++++++++++++++++------------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 380 insertions(+), 135 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 2e37835708..1b2b6ff5c2 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -24,6 +24,7 @@ SQLITE_EXTENSION_INIT1 #include #include #include +#include /* Unsigned integer types */ typedef sqlite3_uint64 u64; @@ -56,12 +57,21 @@ struct Json { #define JSON_ARRAY 6 #define JSON_OBJECT 7 +/* +** Names of the various JSON types: +*/ +static const char * const jsonType[] = { + "null", "true", "false", "integer", "real", "text", "array", "object" +}; + + /* A single node of parsed JSON */ typedef struct JsonNode JsonNode; struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 bRaw; /* Content is raw, rather than JSON encoded */ + u8 bBackslash; /* Formatted JSON_STRING contains \ escapes */ u32 n; /* Bytes of content, or number of sub-nodes */ const char *zJContent; /* JSON content */ }; @@ -268,22 +278,89 @@ static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ sqlite3_result_int(pCtx, 0); break; } - - /* FIXME: We really want to do text->numeric conversion on these. - ** Doing so would be easy if these were internal routines, but the - ** necessary interfaces are not exposed for doing it as a loadable - ** extension. */ - case JSON_REAL: - case JSON_INT: { - sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); + case JSON_REAL: { + double r = strtod(pNode->zJContent, 0); + sqlite3_result_double(pCtx, r); + break; + } + case JSON_INT: { + sqlite3_int64 i = 0; + const char *z = pNode->zJContent; + if( z[0]=='-' ){ z++; } + while( z[0]>='0' && z[0]<='9' ){ i = i*10 + *(z++) - '0'; } + if( pNode->zJContent[0]=='-' ){ i = -i; } + sqlite3_result_int64(pCtx, i); break; } - case JSON_STRING: { if( pNode->bRaw ){ sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); + }else if( !pNode->bBackslash ){ + /* JSON formatted without any backslash-escapes */ + sqlite3_result_text(pCtx, pNode->zJContent+1, pNode->n-2, + SQLITE_TRANSIENT); }else{ /* Translate JSON formatted string into raw text */ + u32 i; + u32 n = pNode->n; + const char *z = pNode->zJContent; + char *zOut; + u32 j; + zOut = sqlite3_malloc( n+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(i=1, j=0; i='0' && c<='9' ) v = v*16 + c - '0'; + else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10; + else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10; + else break; + z++; + } + if( v<=0x7f ){ + zOut[j++] = v; + }else if( v<=0x7ff ){ + zOut[j++] = 0xc0 | (v>>6); + zOut[j++] = 0x80 | (v&0x3f); + }else if( v<=0xffff ){ + zOut[j++] = 0xe0 | (v>>12); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + }else if( v<=0x10ffff ){ + zOut[j++] = 0xf0 | (v>>18); + zOut[j++] = 0x80 | ((v>>12)&0x3f); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + } + }else{ + if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + } + zOut[j++] = c; + } + } + } + zOut[j] = 0; + sqlite3_result_text(pCtx, zOut, j, sqlite3_free); } break; } @@ -298,116 +375,6 @@ static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ } } -/* -** Implementation of the json_array(VALUE,...) function. Return a JSON -** array that contains all values given in arguments. Or if any argument -** is a BLOB, throw an error. -*/ -static void jsonArrayFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int i; - Json jx; - char cSep = '['; - - jsonInit(&jx, context); - for(i=0; iaNode[] of the @@ -440,6 +407,7 @@ static int jsonParseAddNode( p = &pParse->aNode[pParse->nNode]; p->eType = (u8)eType; p->bRaw = 0; + p->bBackslash = 0; p->n = n; p->zJContent = zContent; return pParse->nNode++; @@ -509,6 +477,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ return j+1; }else if( c=='"' ){ /* Parse string */ + u8 bBackslash = 0; j = i+1; for(;;){ c = pParse->zJson[j]; @@ -516,12 +485,14 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c=='\\' ){ c = pParse->zJson[++j]; if( c==0 ) return -1; + bBackslash = 1; }else if( c=='"' ){ break; } j++; } jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]); + if( bBackslash ) pParse->aNode[pParse->nNode-1].bBackslash = 1; return j+1; }else if( c=='n' && strncmp(pParse->zJson+i,"null",4)==0 @@ -601,6 +572,60 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ } return 0; } +/* +** Search along zPath to find the node specified. Return a pointer +** to that node, or NULL if zPath is malformed or if there is no such +** node. +*/ +static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ + u32 i, j; + if( zPath[0]==0 ) return pRoot; + if( zPath[0]=='.' ){ + if( pRoot->eType!=JSON_OBJECT ) return 0; + zPath++; + for(i=0; isalnum(zPath[i]); i++){} + if( i==0 ) return 0; + j = 1; + while( j<=pRoot->n ){ + if( pRoot[j].n==i+2 + && strncmp(&pRoot[j].zJContent[1],zPath,i)==0 + ){ + return jsonLookup(&pRoot[j+1], &zPath[i]); + } + j++; + if( pRoot[j].eType==JSON_ARRAY || pRoot[j].eType==JSON_OBJECT ){ + j += pRoot[j].n; + } + j++; + } + }else if( zPath[0]=='[' && isdigit(zPath[1]) ){ + if( pRoot->eType!=JSON_ARRAY ) return 0; + i = 0; + zPath++; + while( isdigit(zPath[0]) ){ + i = i + zPath[0] - '0'; + zPath++; + } + if( zPath[0]!=']' ) return 0; + zPath++; + j = 1; + while( i>0 && j<=pRoot->n ){ + if( pRoot[j].eType==JSON_ARRAY || pRoot[j].eType==JSON_OBJECT ){ + j += pRoot[j].n; + } + j++; + i--; + } + if( j<=pRoot->n ){ + return jsonLookup(&pRoot[j], zPath); + } + } + return 0; +} + +/**************************************************************************** +** SQL functions used for testing and debugging +****************************************************************************/ /* ** The json_parse(JSON) function returns a string which describes @@ -616,9 +641,6 @@ static void jsonParseFunc( JsonParse x; /* The parse */ u32 i; char zBuf[50]; - static const char *azType[] = { - "NULL", "TRUE", "FALSE", "INT", "REAL", "STRING", "ARRAY", "OBJECT" - }; assert( argc==1 ); if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; @@ -627,7 +649,7 @@ static void jsonParseFunc( sqlite3_snprintf(sizeof(zBuf), zBuf, "node %u:\n", i); jsonAppend(&s, zBuf); sqlite3_snprintf(sizeof(zBuf), zBuf, " type: %s\n", - azType[x.aNode[i].eType]); + jsonType[x.aNode[i].eType]); jsonAppend(&s, zBuf); if( x.aNode[i].eType>=JSON_INT ){ sqlite3_snprintf(sizeof(zBuf), zBuf, " n: %u\n", x.aNode[i].n); @@ -675,6 +697,222 @@ static void jsonNodeCountFunc( sqlite3_free(x.aNode); } +/**************************************************************************** +** SQL function implementations +****************************************************************************/ + +/* +** Implementation of the json_array(VALUE,...) function. Return a JSON +** array that contains all values given in arguments. Or if any argument +** is a BLOB, throw an error. +*/ +static void jsonArrayFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + Json jx; + char cSep = '['; + + jsonInit(&jx, context); + for(i=0; ieType==JSON_ARRAY ){ + for(i=1; i<=pNode->n; i++, n++){ + if( pNode[i].eType==JSON_ARRAY || pNode[i].eType==JSON_OBJECT ){ + i += pNode[i].n; + } + } + } + } + sqlite3_free(x.aNode); + } + sqlite3_result_int64(context, n); +} + +/* +** json_extract(JSON, PATH) +** +** Return the element described by PATH. Return NULL if JSON is not +** valid JSON or if there is no PATH element or if PATH is malformed. +*/ +static void jsonExtractFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + assert( argc==2 ); + zPath = (const char*)sqlite3_value_text(argv[1]); + if( zPath==0 ) return; + if( zPath[0]!='$' ) return; + zPath++; + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + pNode = jsonLookup(x.aNode, zPath); + if( pNode ){ + jsonReturn(pNode, context); + } + sqlite3_free(x.aNode); +} + +/* +** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON +** object that contains all name/value given in arguments. Or if any name +** is not a string or if any value is a BLOB, throw an error. +*/ +static void jsonObjectFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + Json jx; + char cSep = '{'; + const char *z; + u32 n; + + if( argc&1 ){ + sqlite3_result_error(context, "json_object() requires an even number " + "of arguments", -1); + return; + } + jsonInit(&jx, context); + for(i=0; ieType], -1, SQLITE_STATIC); + } + sqlite3_free(x.aNode); +} #ifdef _WIN32 __declspec(dllexport) @@ -691,11 +929,18 @@ int sqlite3_json_init( int nArg; void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { - { "json_array", -1, jsonArrayFunc }, - { "json_object", -1, jsonObjectFunc }, - { "json_parse", 1, jsonParseFunc }, /* DEBUG */ - { "json_test1", 1, jsonTest1Func }, /* DEBUG */ - { "json_nodecount", 1, jsonNodeCountFunc }, /* DEBUG */ + { "json_array", -1, jsonArrayFunc }, + { "json_array_length", 1, jsonArrayLengthFunc }, + { "json_array_length", 2, jsonArrayLengthFunc }, + { "json_extract", 2, jsonExtractFunc }, + { "json_object", -1, jsonObjectFunc }, + { "json_type", 1, jsonTypeFunc }, + { "json_type", 2, jsonTypeFunc }, + + /* DEBUG and TESTING functions */ + { "json_parse", 1, jsonParseFunc }, + { "json_test1", 1, jsonTest1Func }, + { "json_nodecount", 1, jsonNodeCountFunc }, }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ diff --git a/manifest b/manifest index a89a28af9e..67b31769fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Abandon\sthe\sJSONB\sformat\sfor\snow.\s\s(We\smay\sreturn\sto\sit\sin\sthe\sfuture.)\s\sAdd\na\sfunction\sto\srender\sa\sJSON\sparse. -D 2015-08-17T11:28:03.070 +C Initial\simplementation\sfor\sjson_array_length(),\sjson_extract(),\sand\njson_type(). +D 2015-08-17T15:17:37.780 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c fce2fee3ac62dd53cc502cf2673b2ee5947d702d +F ext/misc/json.c f26cbaa8ba1e396b3bf1e29aa116abed2a27ef95 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42c15c1e36b5077646fef99028cf12e587a45023 -R 4717cf337d2cdc65a44f3f6bb72a9c97 +P 9703c0aa18ae43375af876474b818e504e1c10a5 +R cfeb418e7f67bb21a131fd01c852e6d6 U drh -Z 1ecceeb77cb551a38df879c773148819 +Z 903f29a5c61013c958dfe3ed9e5ebef0 diff --git a/manifest.uuid b/manifest.uuid index 669996d2c6..be625468c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9703c0aa18ae43375af876474b818e504e1c10a5 \ No newline at end of file +39983204515837e7bd574cf47918e493acc03d1f \ No newline at end of file From 301eeccaa700ea954abf0c63c2f8526f93d6578e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Aug 2015 20:14:19 +0000 Subject: [PATCH 10/54] Add an initial implementation for json_remove(). FossilOrigin-Name: 2a8267209dbda36a37c1b5f65000b2f763c62341 --- ext/misc/json.c | 148 ++++++++++++++++++++++++++++++++---------------- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 107 insertions(+), 55 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 1b2b6ff5c2..dc27349c0f 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -64,14 +64,19 @@ static const char * const jsonType[] = { "null", "true", "false", "integer", "real", "text", "array", "object" }; +/* Bit values for the JsonNode.jnFlag field +*/ +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ +#define JNODE_REMOVE 0x04 /* Do not output */ + /* A single node of parsed JSON */ typedef struct JsonNode JsonNode; struct JsonNode { u8 eType; /* One of the JSON_ type values */ - u8 bRaw; /* Content is raw, rather than JSON encoded */ - u8 bBackslash; /* Formatted JSON_STRING contains \ escapes */ + u8 jnFlags; /* JNODE flags */ u32 n; /* Bytes of content, or number of sub-nodes */ const char *zJContent; /* JSON content */ }; @@ -87,6 +92,15 @@ struct JsonParse { u8 oom; /* Set to true if out of memory */ }; +/* +** Return the number of consecutive JsonNode slots need to represent +** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and +** OBJECT types, the number might be larger. +*/ +static u32 jsonSizeof(JsonNode *pNode){ + return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; +} + /* Set the Json object to an empty string */ static void jsonZero(Json *p){ @@ -126,7 +140,7 @@ static void jsonOom(Json *p){ ** Return zero on success. Return non-zero on an OOM error */ static int jsonGrow(Json *p, u32 N){ - u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+100; + u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ if( p->oom ) return SQLITE_NOMEM; @@ -171,6 +185,16 @@ static void jsonAppendChar(Json *p, char c){ p->zBuf[p->nUsed++] = c; } +/* Append a comma separator to the output buffer, if the previous +** character is not '[' or '{'. +*/ +static void jsonAppendSeparator(Json *p){ + char c; + if( p->nUsed==0 ) return; + c = p->zBuf[p->nUsed-1]; + if( c!='[' && c!='{' ) jsonAppendChar(p, ','); +} + /* Append the N-byte string in zIn to the end of the Json string ** under construction. Enclose the string in "..." and escape ** any double-quotes or backslash characters contained within the @@ -209,7 +233,7 @@ static void jsonResult(Json *p){ ** the number of JsonNode objects that are encoded. */ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ - u32 j = 0; + u32 j = 1; switch( pNode->eType ){ case JSON_NULL: { jsonAppendRaw(pOut, "null", 4); @@ -224,7 +248,7 @@ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ break; } case JSON_STRING: { - if( pNode->bRaw ){ + if( pNode->jnFlags & JNODE_RAW ){ jsonAppendString(pOut, pNode->zJContent, pNode->n); break; } @@ -237,28 +261,34 @@ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ } case JSON_ARRAY: { jsonAppendChar(pOut, '['); - j = 0; - while( jn ){ - if( j>0 ) jsonAppendChar(pOut, ','); - j += jsonRenderNode(&pNode[j+1], pOut); + while( j<=pNode->n ){ + if( pNode[j].jnFlags & JNODE_REMOVE ){ + j += jsonSizeof(&pNode[j]); + }else{ + jsonAppendSeparator(pOut); + j += jsonRenderNode(&pNode[j], pOut); + } } jsonAppendChar(pOut, ']'); break; } case JSON_OBJECT: { jsonAppendChar(pOut, '{'); - j = 0; - while( jn ){ - if( j>0 ) jsonAppendChar(pOut, ','); - j += jsonRenderNode(&pNode[j+1], pOut); - jsonAppendChar(pOut, ':'); - j += jsonRenderNode(&pNode[j+1], pOut); + while( j<=pNode->n ){ + if( pNode[j+1].jnFlags & JNODE_REMOVE ){ + j += 1 + jsonSizeof(&pNode[j+1]); + }else{ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut); + jsonAppendChar(pOut, ':'); + j += 1 + jsonRenderNode(&pNode[j+1], pOut); + } } jsonAppendChar(pOut, '}'); break; } } - return j+1; + return j; } /* @@ -293,9 +323,9 @@ static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ break; } case JSON_STRING: { - if( pNode->bRaw ){ + if( pNode->jnFlags & JNODE_RAW ){ sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); - }else if( !pNode->bBackslash ){ + }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ /* JSON formatted without any backslash-escapes */ sqlite3_result_text(pCtx, pNode->zJContent+1, pNode->n-2, SQLITE_TRANSIENT); @@ -406,8 +436,7 @@ static int jsonParseAddNode( } p = &pParse->aNode[pParse->nNode]; p->eType = (u8)eType; - p->bRaw = 0; - p->bBackslash = 0; + p->jnFlags = 0; p->n = n; p->zJContent = zContent; return pParse->nNode++; @@ -477,7 +506,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ return j+1; }else if( c=='"' ){ /* Parse string */ - u8 bBackslash = 0; + u8 jnFlags = 0; j = i+1; for(;;){ c = pParse->zJson[j]; @@ -485,14 +514,14 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c=='\\' ){ c = pParse->zJson[++j]; if( c==0 ) return -1; - bBackslash = 1; + jnFlags = JNODE_ESCAPE; }else if( c=='"' ){ break; } j++; } jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]); - if( bBackslash ) pParse->aNode[pParse->nNode-1].bBackslash = 1; + pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; return j+1; }else if( c=='n' && strncmp(pParse->zJson+i,"null",4)==0 @@ -572,6 +601,7 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ } return 0; } + /* ** Search along zPath to find the node specified. Return a pointer ** to that node, or NULL if zPath is malformed or if there is no such @@ -593,10 +623,7 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ return jsonLookup(&pRoot[j+1], &zPath[i]); } j++; - if( pRoot[j].eType==JSON_ARRAY || pRoot[j].eType==JSON_OBJECT ){ - j += pRoot[j].n; - } - j++; + j += jsonSizeof(&pRoot[j]); } }else if( zPath[0]=='[' && isdigit(zPath[1]) ){ if( pRoot->eType!=JSON_ARRAY ) return 0; @@ -610,10 +637,7 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ zPath++; j = 1; while( i>0 && j<=pRoot->n ){ - if( pRoot[j].eType==JSON_ARRAY || pRoot[j].eType==JSON_OBJECT ){ - j += pRoot[j].n; - } - j++; + j += jsonSizeof(&pRoot[j]); i--; } if( j<=pRoot->n ){ @@ -627,6 +651,7 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ ** SQL functions used for testing and debugging ****************************************************************************/ +#ifdef SQLITE_DEBUG /* ** The json_parse(JSON) function returns a string which describes ** a parse of the JSON provided. Or it returns NULL if JSON is not @@ -640,26 +665,17 @@ static void jsonParseFunc( Json s; /* Output string - not real JSON */ JsonParse x; /* The parse */ u32 i; - char zBuf[50]; + char zBuf[100]; assert( argc==1 ); if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; jsonInit(&s, context); for(i=0; i=JSON_INT ){ - sqlite3_snprintf(sizeof(zBuf), zBuf, " n: %u\n", x.aNode[i].n); - jsonAppend(&s, zBuf); - } if( x.aNode[i].zJContent!=0 ){ - sqlite3_snprintf(sizeof(zBuf), zBuf, " ofst: %u\n", - (u32)(x.aNode[i].zJContent - x.zJson)); - jsonAppend(&s, zBuf); - jsonAppendRaw(&s, " text: ", 8); + jsonAppendRaw(&s, " text: ", 10); jsonAppendRaw(&s, x.aNode[i].zJContent, x.aNode[i].n); jsonAppendRaw(&s, "\n", 1); } @@ -696,6 +712,7 @@ static void jsonNodeCountFunc( sqlite3_result_int64(context, x.nNode); sqlite3_free(x.aNode); } +#endif /* SQLITE_DEBUG */ /**************************************************************************** ** SQL function implementations @@ -779,10 +796,8 @@ static void jsonArrayLengthFunc( JsonNode *pNode = x.aNode; if( zPath ) pNode = jsonLookup(pNode, zPath); if( pNode->eType==JSON_ARRAY ){ - for(i=1; i<=pNode->n; i++, n++){ - if( pNode[i].eType==JSON_ARRAY || pNode[i].eType==JSON_OBJECT ){ - i += pNode[i].n; - } + for(i=1; i<=pNode->n; n++){ + i += jsonSizeof(&pNode[i]); } } } @@ -882,6 +897,40 @@ static void jsonObjectFunc( } +/* +** json_remove(JSON, PATH, ...) +** +** Remove the named elements from JSON and return the result. Ill-formed +** PATH arguments are silently ignored. If JSON is ill-formed, then NULL +** is returned. +*/ +static void jsonRemoveFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + + if( argc<1 ) return; + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + if( x.nNode ){ + for(i=1; ijnFlags |= JNODE_REMOVE; + } + if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ + jsonReturn(x.aNode, context); + } + } + sqlite3_free(x.aNode); +} + /* ** json_type(JSON) ** json_type(JSON, PATH) @@ -934,13 +983,16 @@ int sqlite3_json_init( { "json_array_length", 2, jsonArrayLengthFunc }, { "json_extract", 2, jsonExtractFunc }, { "json_object", -1, jsonObjectFunc }, + { "json_remove", -1, jsonRemoveFunc }, { "json_type", 1, jsonTypeFunc }, { "json_type", 2, jsonTypeFunc }, +#if SQLITE_DEBUG /* DEBUG and TESTING functions */ { "json_parse", 1, jsonParseFunc }, { "json_test1", 1, jsonTest1Func }, { "json_nodecount", 1, jsonNodeCountFunc }, +#endif }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ diff --git a/manifest b/manifest index 67b31769fb..4fc2dcacd6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\simplementation\sfor\sjson_array_length(),\sjson_extract(),\sand\njson_type(). -D 2015-08-17T15:17:37.780 +C Add\san\sinitial\simplementation\sfor\sjson_remove(). +D 2015-08-17T20:14:19.276 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c f26cbaa8ba1e396b3bf1e29aa116abed2a27ef95 +F ext/misc/json.c 30fd85ea1fba24031952aa0c635156cfd8ca02ea F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9703c0aa18ae43375af876474b818e504e1c10a5 -R cfeb418e7f67bb21a131fd01c852e6d6 +P 39983204515837e7bd574cf47918e493acc03d1f +R 9ed2e57314d456e476aa0e6edd9d8c55 U drh -Z 903f29a5c61013c958dfe3ed9e5ebef0 +Z 25ffed392c0fb3616f3b027e2b1bb1db diff --git a/manifest.uuid b/manifest.uuid index be625468c7..3cd093bc94 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -39983204515837e7bd574cf47918e493acc03d1f \ No newline at end of file +2a8267209dbda36a37c1b5f65000b2f763c62341 \ No newline at end of file From d096059fca8f84284b8fa66c2e0ad6fa32197750 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Aug 2015 21:22:32 +0000 Subject: [PATCH 11/54] Initial implementation of json_replace(). FossilOrigin-Name: 3c4bee65d93efc7f03f0f11817a068b02068d37c --- ext/misc/json.c | 218 ++++++++++++++++++++++++++++++------------------ manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 145 insertions(+), 87 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index dc27349c0f..d865fb6152 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -42,7 +42,7 @@ struct Json { u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ - u8 oom; /* True if an OOM has been encountered */ + u8 bErr; /* True if an error has been encountered */ char zSpace[100]; /* Initial static space */ }; @@ -69,6 +69,7 @@ static const char * const jsonType[] = { #define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ #define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ #define JNODE_REMOVE 0x04 /* Do not output */ +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ /* A single node of parsed JSON @@ -77,6 +78,7 @@ typedef struct JsonNode JsonNode; struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 jnFlags; /* JNODE flags */ + u8 iVal; /* Replacement value when JNODE_REPLACE */ u32 n; /* Bytes of content, or number of sub-nodes */ const char *zJContent; /* JSON content */ }; @@ -97,7 +99,7 @@ struct JsonParse { ** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and ** OBJECT types, the number might be larger. */ -static u32 jsonSizeof(JsonNode *pNode){ +static u32 jsonSize(JsonNode *pNode){ return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; } @@ -114,7 +116,7 @@ static void jsonZero(Json *p){ */ static void jsonInit(Json *p, sqlite3_context *pCtx){ p->pCtx = pCtx; - p->oom = 0; + p->bErr = 0; jsonZero(p); } @@ -131,9 +133,11 @@ static void jsonReset(Json *p){ /* Report an out-of-memory (OOM) condition */ static void jsonOom(Json *p){ - p->oom = 1; - sqlite3_result_error_nomem(p->pCtx); - jsonReset(p); + if( !p->bErr ){ + p->bErr = 1; + sqlite3_result_error_nomem(p->pCtx); + jsonReset(p); + } } /* Enlarge pJson->zBuf so that it can hold at least N more bytes. @@ -143,7 +147,7 @@ static int jsonGrow(Json *p, u32 N){ u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ - if( p->oom ) return SQLITE_NOMEM; + if( p->bErr ) return 1; zNew = sqlite3_malloc64(nTotal); if( zNew==0 ){ jsonOom(p); @@ -172,11 +176,13 @@ static void jsonAppendRaw(Json *p, const char *zIn, u32 N){ p->nUsed += N; } +#ifdef SQLITE_DEBUG /* Append the zero-terminated string zIn */ static void jsonAppend(Json *p, const char *zIn){ jsonAppendRaw(p, zIn, (u32)strlen(zIn)); } +#endif /* Append a single character */ @@ -215,10 +221,48 @@ static void jsonAppendString(Json *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = '"'; } +/* +** Append a function parameter value to the JSON string under +** construction. +*/ +static void jsonAppendValue( + Json *p, /* Append to this JSON string */ + sqlite3_value *pValue /* Value to append */ +){ + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonAppendRaw(p, "null", 4); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + jsonAppendRaw(p, z, n); + break; + } + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + jsonAppendString(p, z, n); + break; + } + default: { + if( p->bErr==0 ){ + sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); + p->bErr = 1; + jsonReset(p); + } + break; + } + } +} + + /* Make the JSON in p the result of the SQL function. */ static void jsonResult(Json *p){ - if( p->oom==0 ){ + if( p->bErr==0 ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, SQLITE_UTF8); @@ -232,7 +276,11 @@ static void jsonResult(Json *p){ ** append to pOut. Subsubstructure is also included. Return ** the number of JsonNode objects that are encoded. */ -static int jsonRenderNode(JsonNode *pNode, Json *pOut){ +static int jsonRenderNode( + JsonNode *pNode, /* The node to render */ + Json *pOut, /* Write JSON here */ + sqlite3_value **aReplace /* Replacement values */ +){ u32 j = 1; switch( pNode->eType ){ case JSON_NULL: { @@ -262,11 +310,15 @@ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ case JSON_ARRAY: { jsonAppendChar(pOut, '['); while( j<=pNode->n ){ - if( pNode[j].jnFlags & JNODE_REMOVE ){ - j += jsonSizeof(&pNode[j]); + if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ + if( pNode[j].jnFlags & JNODE_REPLACE ){ + jsonAppendSeparator(pOut); + jsonAppendValue(pOut, aReplace[pNode[j].iVal]); + } + j += jsonSize(&pNode[j]); }else{ jsonAppendSeparator(pOut); - j += jsonRenderNode(&pNode[j], pOut); + j += jsonRenderNode(&pNode[j], pOut, aReplace); } } jsonAppendChar(pOut, ']'); @@ -276,12 +328,17 @@ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ jsonAppendChar(pOut, '{'); while( j<=pNode->n ){ if( pNode[j+1].jnFlags & JNODE_REMOVE ){ - j += 1 + jsonSizeof(&pNode[j+1]); + j += 1 + jsonSize(&pNode[j+1]); }else{ jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); jsonAppendChar(pOut, ':'); - j += 1 + jsonRenderNode(&pNode[j+1], pOut); + if( pNode[j+1].jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); + j += 1 + jsonSize(&pNode[j+1]); + }else{ + j += 1 + jsonRenderNode(&pNode[j+1], pOut, aReplace); + } } } jsonAppendChar(pOut, '}'); @@ -294,7 +351,11 @@ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ /* ** Make the JsonNode the return value of the function. */ -static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ +static void jsonReturn( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ switch( pNode->eType ){ case JSON_NULL: { sqlite3_result_null(pCtx); @@ -398,7 +459,7 @@ static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ case JSON_OBJECT: { Json s; jsonInit(&s, pCtx); - jsonRenderNode(pNode, &s); + jsonRenderNode(pNode, &s, aReplace); jsonResult(&s); break; } @@ -437,6 +498,7 @@ static int jsonParseAddNode( p = &pParse->aNode[pParse->nNode]; p->eType = (u8)eType; p->jnFlags = 0; + p->iVal = 0; p->n = n; p->zJContent = zContent; return pParse->nNode++; @@ -623,7 +685,7 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ return jsonLookup(&pRoot[j+1], &zPath[i]); } j++; - j += jsonSizeof(&pRoot[j]); + j += jsonSize(&pRoot[j]); } }else if( zPath[0]=='[' && isdigit(zPath[1]) ){ if( pRoot->eType!=JSON_ARRAY ) return 0; @@ -637,7 +699,7 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ zPath++; j = 1; while( i>0 && j<=pRoot->n ){ - j += jsonSizeof(&pRoot[j]); + j += jsonSize(&pRoot[j]); i--; } if( j<=pRoot->n ){ @@ -694,7 +756,7 @@ static void jsonTest1Func( ){ JsonParse x; /* The parse */ if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonReturn(x.aNode, context); + jsonReturn(x.aNode, context, 0); sqlite3_free(x.aNode); } @@ -730,38 +792,14 @@ static void jsonArrayFunc( ){ int i; Json jx; - char cSep = '['; jsonInit(&jx, context); + jsonAppendChar(&jx, '['); for(i=0; ieType==JSON_ARRAY ){ for(i=1; i<=pNode->n; n++){ - i += jsonSizeof(&pNode[i]); + i += jsonSize(&pNode[i]); } } } @@ -828,7 +866,7 @@ static void jsonExtractFunc( if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; pNode = jsonLookup(x.aNode, zPath); if( pNode ){ - jsonReturn(pNode, context); + jsonReturn(pNode, context, 0); } sqlite3_free(x.aNode); } @@ -845,7 +883,6 @@ static void jsonObjectFunc( ){ int i; Json jx; - char cSep = '{'; const char *z; u32 n; @@ -855,44 +892,21 @@ static void jsonObjectFunc( return; } jsonInit(&jx, context); + jsonAppendChar(&jx, '{'); for(i=0; ijnFlags |= JNODE_REMOVE; } if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturn(x.aNode, context); + jsonReturn(x.aNode, context, 0); + } + } + sqlite3_free(x.aNode); +} + +/* +** json_replace(JSON, PATH, VALUE, ...) +** +** Replace the value at PATH with VALUE. If PATH does not already exist, +** this routine is a no-op. If JSON is ill-formed, return NULL. +*/ +static void jsonReplaceFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + + if( argc<1 ) return; + if( (argc&1)==0 ) { + sqlite3_result_error(context, + "json_replace() needs an odd number of arguments", -1); + return; + } + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + if( x.nNode ){ + for(i=1; ijnFlags |= JNODE_REPLACE; + pNode->iVal = i+1; + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(context, argv[x.aNode[0].iVal]); + }else{ + jsonReturn(x.aNode, context, argv); } } sqlite3_free(x.aNode); @@ -984,6 +1041,7 @@ int sqlite3_json_init( { "json_extract", 2, jsonExtractFunc }, { "json_object", -1, jsonObjectFunc }, { "json_remove", -1, jsonRemoveFunc }, + { "json_replace", -1, jsonReplaceFunc }, { "json_type", 1, jsonTypeFunc }, { "json_type", 2, jsonTypeFunc }, diff --git a/manifest b/manifest index 4fc2dcacd6..7d15376185 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sinitial\simplementation\sfor\sjson_remove(). -D 2015-08-17T20:14:19.276 +C Initial\simplementation\sof\sjson_replace(). +D 2015-08-17T21:22:32.495 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 30fd85ea1fba24031952aa0c635156cfd8ca02ea +F ext/misc/json.c d96116de8aafdb117b99712b2a83144d86755350 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 39983204515837e7bd574cf47918e493acc03d1f -R 9ed2e57314d456e476aa0e6edd9d8c55 +P 2a8267209dbda36a37c1b5f65000b2f763c62341 +R f98c8d887493a196dac50851b7b4599f U drh -Z 25ffed392c0fb3616f3b027e2b1bb1db +Z 432e296535146c125ab741ef125c263a diff --git a/manifest.uuid b/manifest.uuid index 3cd093bc94..e0120e1b2e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2a8267209dbda36a37c1b5f65000b2f763c62341 \ No newline at end of file +3c4bee65d93efc7f03f0f11817a068b02068d37c \ No newline at end of file From 52216adf7efa4b7ebb3306c9a7a92c20e087015c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Aug 2015 02:28:03 +0000 Subject: [PATCH 12/54] Initial implementation of json_set() and json_insert(). FossilOrigin-Name: 4aa49656d98e2894f2faa8963f79462ee6165d40 --- ext/misc/json.c | 276 ++++++++++++++++++++++++++++++++++++------------ manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 215 insertions(+), 75 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index d865fb6152..e980b52de6 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -31,11 +31,15 @@ typedef sqlite3_uint64 u64; typedef unsigned int u32; typedef unsigned char u8; +/* Objects */ +typedef struct Json Json; +typedef struct JsonNode JsonNode; +typedef struct JsonParse JsonParse; + /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator ** that can be and is used to create strings other than JSON. */ -typedef struct Json Json; struct Json { sqlite3_context *pCtx; /* Function context - put error messages here */ char *zBuf; /* Append JSON content here */ @@ -70,22 +74,24 @@ static const char * const jsonType[] = { #define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ #define JNODE_REMOVE 0x04 /* Do not output */ #define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ +#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ /* A single node of parsed JSON */ -typedef struct JsonNode JsonNode; struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 jnFlags; /* JNODE flags */ u8 iVal; /* Replacement value when JNODE_REPLACE */ u32 n; /* Bytes of content, or number of sub-nodes */ - const char *zJContent; /* JSON content */ + union { + const char *zJContent; /* JSON content */ + u32 iAppend; /* Appended content */ + } u; }; /* A completely parsed JSON string */ -typedef struct JsonParse JsonParse; struct JsonParse { u32 nNode; /* Number of slots of aNode[] used */ u32 nAlloc; /* Number of slots of aNode[] allocated */ @@ -98,6 +104,10 @@ struct JsonParse { ** Return the number of consecutive JsonNode slots need to represent ** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and ** OBJECT types, the number might be larger. +** +** Appended elements are not counted. The value returned is the number +** by which the JsonNode counter should increment in order to go to the +** next peer value. */ static u32 jsonSize(JsonNode *pNode){ return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; @@ -276,12 +286,11 @@ static void jsonResult(Json *p){ ** append to pOut. Subsubstructure is also included. Return ** the number of JsonNode objects that are encoded. */ -static int jsonRenderNode( +static void jsonRenderNode( JsonNode *pNode, /* The node to render */ Json *pOut, /* Write JSON here */ sqlite3_value **aReplace /* Replacement values */ ){ - u32 j = 1; switch( pNode->eType ){ case JSON_NULL: { jsonAppendRaw(pOut, "null", 4); @@ -297,55 +306,64 @@ static int jsonRenderNode( } case JSON_STRING: { if( pNode->jnFlags & JNODE_RAW ){ - jsonAppendString(pOut, pNode->zJContent, pNode->n); + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); break; } /* Fall through into the next case */ } case JSON_REAL: case JSON_INT: { - jsonAppendRaw(pOut, pNode->zJContent, pNode->n); + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); break; } case JSON_ARRAY: { + u32 j = 1; jsonAppendChar(pOut, '['); - while( j<=pNode->n ){ - if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ - if( pNode[j].jnFlags & JNODE_REPLACE ){ + for(;;){ + while( j<=pNode->n ){ + if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ + if( pNode[j].jnFlags & JNODE_REPLACE ){ + jsonAppendSeparator(pOut); + jsonAppendValue(pOut, aReplace[pNode[j].iVal]); + } + }else{ jsonAppendSeparator(pOut); - jsonAppendValue(pOut, aReplace[pNode[j].iVal]); + jsonRenderNode(&pNode[j], pOut, aReplace); } j += jsonSize(&pNode[j]); - }else{ - jsonAppendSeparator(pOut); - j += jsonRenderNode(&pNode[j], pOut, aReplace); } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; } jsonAppendChar(pOut, ']'); break; } case JSON_OBJECT: { + u32 j = 1; jsonAppendChar(pOut, '{'); - while( j<=pNode->n ){ - if( pNode[j+1].jnFlags & JNODE_REMOVE ){ - j += 1 + jsonSize(&pNode[j+1]); - }else{ - jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); - jsonAppendChar(pOut, ':'); - if( pNode[j+1].jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); - j += 1 + jsonSize(&pNode[j+1]); - }else{ - j += 1 + jsonRenderNode(&pNode[j+1], pOut, aReplace); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + jsonAppendChar(pOut, ':'); + if( pNode[j+1].jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); + }else{ + jsonRenderNode(&pNode[j+1], pOut, aReplace); + } } + j += 1 + jsonSize(&pNode[j+1]); } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; } jsonAppendChar(pOut, '}'); break; } } - return j; } /* @@ -370,31 +388,32 @@ static void jsonReturn( break; } case JSON_REAL: { - double r = strtod(pNode->zJContent, 0); + double r = strtod(pNode->u.zJContent, 0); sqlite3_result_double(pCtx, r); break; } case JSON_INT: { sqlite3_int64 i = 0; - const char *z = pNode->zJContent; + const char *z = pNode->u.zJContent; if( z[0]=='-' ){ z++; } while( z[0]>='0' && z[0]<='9' ){ i = i*10 + *(z++) - '0'; } - if( pNode->zJContent[0]=='-' ){ i = -i; } + if( pNode->u.zJContent[0]=='-' ){ i = -i; } sqlite3_result_int64(pCtx, i); break; } case JSON_STRING: { if( pNode->jnFlags & JNODE_RAW ){ - sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, + SQLITE_TRANSIENT); }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ /* JSON formatted without any backslash-escapes */ - sqlite3_result_text(pCtx, pNode->zJContent+1, pNode->n-2, + sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, SQLITE_TRANSIENT); }else{ /* Translate JSON formatted string into raw text */ u32 i; u32 n = pNode->n; - const char *z = pNode->zJContent; + const char *z = pNode->u.zJContent; char *zOut; u32 j; zOut = sqlite3_malloc( n+1 ); @@ -500,7 +519,7 @@ static int jsonParseAddNode( p->jnFlags = 0; p->iVal = 0; p->n = n; - p->zJContent = zContent; + p->u.zJContent = zContent; return pParse->nNode++; } @@ -664,13 +683,26 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ return 0; } +/* forward declaration */ +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*); + /* ** Search along zPath to find the node specified. Return a pointer ** to that node, or NULL if zPath is malformed or if there is no such ** node. +** +** If pApnd!=0, then try to append new nodes to complete zPath if it is +** possible to do so and if no existing node corresponds to zPath. If +** new nodes are appended *pApnd is set to 1. */ -static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ - u32 i, j; +static JsonNode *jsonLookup( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this node */ + const char *zPath, /* The path to search */ + int *pApnd /* Append nodes to complete path if not NULL */ +){ + u32 i, j, k; + JsonNode *pRoot = &pParse->aNode[iRoot]; if( zPath[0]==0 ) return pRoot; if( zPath[0]=='.' ){ if( pRoot->eType!=JSON_OBJECT ) return 0; @@ -678,14 +710,29 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ for(i=0; isalnum(zPath[i]); i++){} if( i==0 ) return 0; j = 1; - while( j<=pRoot->n ){ - if( pRoot[j].n==i+2 - && strncmp(&pRoot[j].zJContent[1],zPath,i)==0 - ){ - return jsonLookup(&pRoot[j+1], &zPath[i]); + for(;;){ + while( j<=pRoot->n ){ + if( pRoot[j].n==i+2 + && strncmp(&pRoot[j].u.zJContent[1],zPath,i)==0 + ){ + return jsonLookup(pParse, iRoot+j+1, &zPath[i], pApnd); + } + j++; + j += jsonSize(&pRoot[j]); } - j++; - j += jsonSize(&pRoot[j]); + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( pApnd ){ + k = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + pRoot->u.iAppend = k - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + k = jsonParseAddNode(pParse, JSON_STRING, i, zPath); + if( !pParse->oom ) pParse->aNode[k].jnFlags |= JNODE_RAW; + zPath += i; + return jsonLookupAppend(pParse, zPath, pApnd); } }else if( zPath[0]=='[' && isdigit(zPath[1]) ){ if( pRoot->eType!=JSON_ARRAY ) return 0; @@ -698,17 +745,54 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ if( zPath[0]!=']' ) return 0; zPath++; j = 1; - while( i>0 && j<=pRoot->n ){ - j += jsonSize(&pRoot[j]); - i--; + for(;;){ + while( i>0 && j<=pRoot->n ){ + j += jsonSize(&pRoot[j]); + i--; + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; } if( j<=pRoot->n ){ - return jsonLookup(&pRoot[j], zPath); + return jsonLookup(pParse, iRoot+j, zPath, pApnd); + } + if( i==0 && pApnd ){ + k = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); + pRoot->u.iAppend = k - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + return jsonLookupAppend(pParse, zPath, pApnd); } } return 0; } +/* +** Append content to pParse that will complete zPath. +*/ +static JsonNode *jsonLookupAppend( + JsonParse *pParse, /* Append content to the JSON parse */ + const char *zPath, /* Description of content to append */ + int *pApnd /* Set this flag to 1 */ +){ + *pApnd = 1; + if( zPath[0]==0 ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; + } + if( zPath[0]=='.' ){ + jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + }else if( strncmp(zPath,"[0]",3)==0 ){ + jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + }else{ + return 0; + } + if( pParse->oom ) return 0; + return jsonLookup(pParse, pParse->nNode-1, zPath, pApnd); +} + + /**************************************************************************** ** SQL functions used for testing and debugging ****************************************************************************/ @@ -736,9 +820,9 @@ static void jsonParseFunc( sqlite3_snprintf(sizeof(zBuf), zBuf, "node %3u: %7s n=%d\n", i, jsonType[x.aNode[i].eType], x.aNode[i].n); jsonAppend(&s, zBuf); - if( x.aNode[i].zJContent!=0 ){ + if( x.aNode[i].u.zJContent!=0 ){ jsonAppendRaw(&s, " text: ", 10); - jsonAppendRaw(&s, x.aNode[i].zJContent, x.aNode[i].n); + jsonAppendRaw(&s, x.aNode[i].u.zJContent, x.aNode[i].n); jsonAppendRaw(&s, "\n", 1); } } @@ -832,8 +916,9 @@ static void jsonArrayLengthFunc( if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0]))==0 ){ if( x.nNode ){ JsonNode *pNode = x.aNode; - if( zPath ) pNode = jsonLookup(pNode, zPath); + if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); if( pNode->eType==JSON_ARRAY ){ + assert( (pNode->jnFlags & JNODE_APPEND)==0 ); for(i=1; i<=pNode->n; n++){ i += jsonSize(&pNode[i]); } @@ -864,7 +949,7 @@ static void jsonExtractFunc( if( zPath[0]!='$' ) return; zPath++; if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; - pNode = jsonLookup(x.aNode, zPath); + pNode = jsonLookup(&x, 0, zPath, 0); if( pNode ){ jsonReturn(pNode, context, 0); } @@ -935,7 +1020,7 @@ static void jsonRemoveFunc( zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) continue; if( zPath[0]!='$' ) continue; - pNode = jsonLookup(x.aNode, &zPath[1]); + pNode = jsonLookup(&x, 0, &zPath[1], 0); if( pNode ) pNode->jnFlags |= JNODE_REMOVE; } if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ @@ -973,7 +1058,7 @@ static void jsonReplaceFunc( zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) continue; if( zPath[0]!='$' ) continue; - pNode = jsonLookup(x.aNode, &zPath[1]); + pNode = jsonLookup(&x, 0, &zPath[1], 0); if( pNode ){ pNode->jnFlags |= JNODE_REPLACE; pNode->iVal = i+1; @@ -987,6 +1072,57 @@ static void jsonReplaceFunc( } sqlite3_free(x.aNode); } +/* +** json_set(JSON, PATH, VALUE, ...) +** +** Set the value at PATH to VALUE. Create the PATH if it does not already +** exist. Overwrite existing values that do exist. +** If JSON is ill-formed, return NULL. +** +** json_insert(JSON, PATH, VALUE, ...) +** +** Create PATH and initialize it to VALUE. If PATH already exists, this +** routine is a no-op. If JSON is ill-formed, return NULL. +*/ +static void jsonSetFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + int bApnd; + int bIsSet = *(int*)sqlite3_user_data(context); + + if( argc<1 ) return; + if( (argc&1)==0 ) { + sqlite3_result_error(context, + "json_set() needs an odd number of arguments", -1); + return; + } + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + if( x.nNode ){ + for(i=1; ijnFlags |= JNODE_REPLACE; + pNode->iVal = i+1; + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(context, argv[x.aNode[0].iVal]); + }else{ + jsonReturn(x.aNode, context, argv); + } + } + sqlite3_free(x.aNode); +} /* ** json_type(JSON) @@ -1014,7 +1150,7 @@ static void jsonTypeFunc( if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ JsonNode *pNode = x.aNode; - if( zPath ) pNode = jsonLookup(pNode, zPath); + if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); sqlite3_result_text(context, jsonType[pNode->eType], -1, SQLITE_STATIC); } sqlite3_free(x.aNode); @@ -1033,30 +1169,34 @@ int sqlite3_json_init( static const struct { const char *zName; int nArg; + int flag; void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { - { "json_array", -1, jsonArrayFunc }, - { "json_array_length", 1, jsonArrayLengthFunc }, - { "json_array_length", 2, jsonArrayLengthFunc }, - { "json_extract", 2, jsonExtractFunc }, - { "json_object", -1, jsonObjectFunc }, - { "json_remove", -1, jsonRemoveFunc }, - { "json_replace", -1, jsonReplaceFunc }, - { "json_type", 1, jsonTypeFunc }, - { "json_type", 2, jsonTypeFunc }, + { "json_array", -1, 0, jsonArrayFunc }, + { "json_array_length", 1, 0, jsonArrayLengthFunc }, + { "json_array_length", 2, 0, jsonArrayLengthFunc }, + { "json_extract", 2, 0, jsonExtractFunc }, + { "json_insert", -1, 0, jsonSetFunc }, + { "json_object", -1, 0, jsonObjectFunc }, + { "json_remove", -1, 0, jsonRemoveFunc }, + { "json_replace", -1, 0, jsonReplaceFunc }, + { "json_set", -1, 1, jsonSetFunc }, + { "json_type", 1, 0, jsonTypeFunc }, + { "json_type", 2, 0, jsonTypeFunc }, #if SQLITE_DEBUG /* DEBUG and TESTING functions */ - { "json_parse", 1, jsonParseFunc }, - { "json_test1", 1, jsonTest1Func }, - { "json_nodecount", 1, jsonNodeCountFunc }, + { "json_parse", 1, 0, jsonParseFunc }, + { "json_test1", 1, 0, jsonTest1Func }, + { "json_nodecount", 1, 0, jsonNodeCountFunc }, #endif }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ for(i=0; i Date: Tue, 18 Aug 2015 12:59:58 +0000 Subject: [PATCH 13/54] Comment clarification. No changes to code. FossilOrigin-Name: 71a966952cf8f1d699cf8362ea3f6204c6e16384 --- ext/misc/json.c | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index e980b52de6..274749726d 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -85,8 +85,8 @@ struct JsonNode { u8 iVal; /* Replacement value when JNODE_REPLACE */ u32 n; /* Bytes of content, or number of sub-nodes */ union { - const char *zJContent; /* JSON content */ - u32 iAppend; /* Appended content */ + const char *zJContent; /* Content for INT, REAL, and STRING */ + u32 iAppend; /* More terms for ARRAY and OBJECT */ } u; }; diff --git a/manifest b/manifest index 9469d276c7..c69e955bf7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\simplementation\sof\sjson_set()\sand\sjson_insert(). -D 2015-08-18T02:28:03.829 +C Comment\sclarification.\s\sNo\schanges\sto\scode. +D 2015-08-18T12:59:58.393 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c d4e2e960549ec023e56f74ee72343416f5f9ef17 +F ext/misc/json.c 3b265167a9f8f006042bf0226a838c773a098c03 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3c4bee65d93efc7f03f0f11817a068b02068d37c -R 1a1455dd49cebbaac366ccfaac1c0acb +P 4aa49656d98e2894f2faa8963f79462ee6165d40 +R c3c28209f7dd1255986fe872c35f7d65 U drh -Z 7cdd28284db3ca2451d1af500f69309d +Z 63f1a876b018989d4396ee5737a9cb33 diff --git a/manifest.uuid b/manifest.uuid index bb7757d569..787f49ef02 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4aa49656d98e2894f2faa8963f79462ee6165d40 \ No newline at end of file +71a966952cf8f1d699cf8362ea3f6204c6e16384 \ No newline at end of file From cb6c6c6f45955918e2b73653ae7c6676288de0b2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Aug 2015 22:47:17 +0000 Subject: [PATCH 14/54] Add the json_each(JSON,PATH) table-valued-function. FossilOrigin-Name: 3335ac17bbcb09dc915173d69bf42048f84ad563 --- ext/misc/json.c | 277 +++++++++++++++++++++++++++++++++++++++++++++++- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 283 insertions(+), 8 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 274749726d..7e1d210e37 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -17,7 +17,9 @@ ** ** For the time being, all JSON is stored as pure text. (We might add ** a JSONB type in the future which stores a binary encoding of JSON in -** a BLOB, but there is no support for JSONB in the current implementation.) +** a BLOB, but there is no support for JSONB in the current implementation. +** This implementation parses JSON text at 250 MB/s, so it is hard to see +** how JSONB might improve on that.) */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -1156,6 +1158,276 @@ static void jsonTypeFunc( sqlite3_free(x.aNode); } +/**************************************************************************** +** The json_each virtual table +****************************************************************************/ +typedef struct JsonEachCursor JsonEachCursor; +struct JsonEachCursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + u32 iRowid; /* The rowid */ + u32 i; /* Index in sParse.aNode[] of current row */ + u32 iEnd; /* EOF when i equals or exceeds this value */ + u8 eType; /* Type of top-level element */ + char *zJson; /* Input json */ + char *zPath; /* Path by which to filter zJson */ + JsonParse sParse; /* The input json */ +}; + +/* Constructor for the json_each virtual table */ +static int jsonEachConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + +/* Column numbers */ +#define JEACH_KEY 0 +#define JEACH_VALUE 1 +#define JEACH_JSON 2 +#define JEACH_PATH 3 + + sqlite3_declare_vtab(db, "CREATE TABLE x(key,value,json hidden,path hidden)"); + memset(pNew, 0, sizeof(*pNew)); + return SQLITE_OK; +} + +/* destructor for json_each virtual table */ +static int jsonEachDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object. */ +static int jsonEachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachCursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* Reset a JsonEachCursor back to its original state. Free any memory +** held. */ +static void jsonEachCursorReset(JsonEachCursor *p){ + sqlite3_free(p->zJson); + sqlite3_free(p->zPath); + sqlite3_free(p->sParse.aNode); + p->iRowid = 0; + p->i = 0; + p->iEnd = 0; + p->eType = 0; + memset(&p->sParse, 0, sizeof(p->sParse)); + p->zJson = 0; + p->zPath = 0; +} + +/* Destructor for a jsonEachCursor object */ +static int jsonEachClose(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + jsonEachCursorReset(p); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* Return TRUE if the jsonEachCursor object has been advanced off the end +** of the JSON object */ +static int jsonEachEof(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + return p->i >= p->iEnd; +} + +/* Advance the cursor to the next top-level element of the current +** JSON string */ +static int jsonEachNext(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + switch( p->eType ){ + case JSON_ARRAY: { + p->i += jsonSize(&p->sParse.aNode[p->i]); + p->iRowid++; + break; + } + case JSON_OBJECT: { + p->i += 1 + jsonSize(&p->sParse.aNode[p->i+1]); + p->iRowid++; + break; + } + default: { + p->i = p->iEnd; + break; + } + } + return SQLITE_OK; +} + +/* Return the value of a column */ +static int jsonEachColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + switch( i ){ + case JEACH_KEY: { + if( p->eType==JSON_OBJECT ){ + jsonReturn(&p->sParse.aNode[p->i], ctx, 0); + }else{ + sqlite3_result_int64(ctx, p->iRowid); + } + break; + } + case JEACH_VALUE: { + if( p->eType==JSON_OBJECT ){ + jsonReturn(&p->sParse.aNode[p->i+1], ctx, 0); + }else{ + jsonReturn(&p->sParse.aNode[p->i], ctx, 0); + } + break; + } + case JEACH_PATH: { + const char *zPath = p->zPath; + if( zPath==0 ) zPath = "$"; + sqlite3_result_text(ctx, zPath, -1, SQLITE_STATIC); + break; + } + default: { + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +/* Return the current rowid value */ +static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + JsonEachCursor *p = (JsonEachCursor*)cur; + *pRowid = p->iRowid; + return SQLITE_OK; +} + +/* The query strategy is to look for an equality constraint on the json +** column. Without such a constraint, the table cannot operate. idxNum is +** 1 if the constraint is found, 3 if the constraint and zPath are found, +** and 0 otherwise. +*/ +static int jsonEachBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + int jsonIdx = -1; + int pathIdx = -1; + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case JEACH_JSON: jsonIdx = i; break; + case JEACH_PATH: pathIdx = i; break; + default: /* no-op */ break; + } + } + if( jsonIdx<0 ){ + pIdxInfo->idxNum = 0; + pIdxInfo->estimatedCost = (double)2000000000; + }else{ + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1; + pIdxInfo->aConstraintUsage[jsonIdx].omit = 1; + if( pathIdx<0 ){ + pIdxInfo->idxNum = 1; + }else{ + pIdxInfo->aConstraintUsage[pathIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[pathIdx].omit = 1; + pIdxInfo->idxNum = 3; + } + } + return SQLITE_OK; +} + +/* Start a search on a new JSON string */ +static int jsonEachFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + const char *z; + const char *zPath; + sqlite3_int64 n; + + jsonEachCursorReset(p); + if( idxNum==0 ) return SQLITE_OK; + z = (const char*)sqlite3_value_text(argv[0]); + if( z==0 ) return SQLITE_OK; + if( idxNum&2 ){ + zPath = (const char*)sqlite3_value_text(argv[1]); + if( zPath==0 || zPath[0]!='$' ) return SQLITE_OK; + } + n = sqlite3_value_bytes(argv[0]); + p->zJson = sqlite3_malloc( n+1 ); + if( p->zJson==0 ) return SQLITE_NOMEM; + memcpy(p->zJson, z, n+1); + if( jsonParse(&p->sParse, p->zJson) ){ + jsonEachCursorReset(p); + }else{ + JsonNode *pNode; + if( idxNum==3 ){ + n = sqlite3_value_bytes(argv[1]); + p->zPath = sqlite3_malloc( n+1 ); + if( p->zPath==0 ) return SQLITE_NOMEM; + memcpy(p->zPath, zPath, n+1); + pNode = jsonLookup(&p->sParse, 0, p->zPath+1, 0); + if( pNode==0 ){ + jsonEachCursorReset(p); + return SQLITE_OK; + } + }else{ + pNode = p->sParse.aNode; + } + p->i = (int)(pNode - p->sParse.aNode); + p->eType = pNode->eType; + if( p->eType>=JSON_ARRAY ){ + p->i++; + p->iEnd = p->i + pNode->n; + }else{ + p->iEnd = p->i+1; + } + } + return SQLITE_OK; +} + +/* The methods of the json_each virtual table */ +static sqlite3_module jsonEachModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpen, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + + #ifdef _WIN32 __declspec(dllexport) #endif @@ -1199,5 +1471,8 @@ int sqlite3_json_init( (void*)&aFunc[i].flag, aFunc[i].xFunc, 0, 0); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "json_each", &jsonEachModule, 0); + } return rc; } diff --git a/manifest b/manifest index fa1a720a00..d5dc07e75e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\stable-valued-function\srowid\sfix. -D 2015-08-19T19:26:13.526 +C Add\sthe\sjson_each(JSON,PATH)\stable-valued-function. +D 2015-08-19T22:47:17.476 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 90f3097efb9a53f5fc59a4f8a08be07cf9f52c02 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 3b265167a9f8f006042bf0226a838c773a098c03 +F ext/misc/json.c 57a9f747b2813edc5f481eb808889d304a1c5f79 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 96a5d44d9fcb7b159c04630ad2d956fe27df5a43 a325a08599759471047e234ef9cfcc3cb110aafd -R dcfe96af8a1b65c426e62b553bd60db8 +P a06a6392bd48baa8b9bac2624869c0cc7da7e779 +R 43d8c9f353202a0c5a0252660436f9ac U drh -Z 00eb910bc67976561bafd5a76ff4be3d +Z 41b78b71d0425818407424da18ee9720 diff --git a/manifest.uuid b/manifest.uuid index ab65dd6ddb..a1d2dfbb42 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a06a6392bd48baa8b9bac2624869c0cc7da7e779 \ No newline at end of file +3335ac17bbcb09dc915173d69bf42048f84ad563 \ No newline at end of file From 6b43cc85318e15be7e42e4da8fab7024fd626ed3 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Aug 2015 23:02:49 +0000 Subject: [PATCH 15/54] Fix the path lookup for objects so that it can handle quoted identifier names and non-alphanumerics in the identifier. FossilOrigin-Name: 87f5873004f69396baa7c67937342b4e32f79bda --- ext/misc/json.c | 20 +++++++++++++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 7e1d210e37..77bc6d6794 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -703,19 +703,29 @@ static JsonNode *jsonLookup( const char *zPath, /* The path to search */ int *pApnd /* Append nodes to complete path if not NULL */ ){ - u32 i, j, k; + u32 i, j, k, nKey; + const char *zKey; JsonNode *pRoot = &pParse->aNode[iRoot]; if( zPath[0]==0 ) return pRoot; if( zPath[0]=='.' ){ if( pRoot->eType!=JSON_OBJECT ) return 0; zPath++; - for(i=0; isalnum(zPath[i]); i++){} - if( i==0 ) return 0; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ) i++; + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + } + if( nKey==0 ) return 0; j = 1; for(;;){ while( j<=pRoot->n ){ - if( pRoot[j].n==i+2 - && strncmp(&pRoot[j].u.zJContent[1],zPath,i)==0 + if( pRoot[j].n==nKey+2 + && strncmp(&pRoot[j].u.zJContent[1],zKey,nKey)==0 ){ return jsonLookup(pParse, iRoot+j+1, &zPath[i], pApnd); } diff --git a/manifest b/manifest index d5dc07e75e..28efbe5685 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sjson_each(JSON,PATH)\stable-valued-function. -D 2015-08-19T22:47:17.476 +C Fix\sthe\spath\slookup\sfor\sobjects\sso\sthat\sit\scan\shandle\squoted\sidentifier\nnames\sand\snon-alphanumerics\sin\sthe\sidentifier. +D 2015-08-19T23:02:49.082 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 90f3097efb9a53f5fc59a4f8a08be07cf9f52c02 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 57a9f747b2813edc5f481eb808889d304a1c5f79 +F ext/misc/json.c 8654245a5d5aeb499fb0c00abce53f3836d52c20 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a06a6392bd48baa8b9bac2624869c0cc7da7e779 -R 43d8c9f353202a0c5a0252660436f9ac +P 3335ac17bbcb09dc915173d69bf42048f84ad563 +R bd9ac56614e747436a95e2948b6eea19 U drh -Z 41b78b71d0425818407424da18ee9720 +Z b6a64887dfaaae356da3f82cf4115061 diff --git a/manifest.uuid b/manifest.uuid index a1d2dfbb42..40c8eba985 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3335ac17bbcb09dc915173d69bf42048f84ad563 \ No newline at end of file +87f5873004f69396baa7c67937342b4e32f79bda \ No newline at end of file From 0dfbe064de551d09ecf7cd82463ac418a0b50566 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Aug 2015 12:37:49 +0000 Subject: [PATCH 16/54] Fix typo in comment. No changes to code. FossilOrigin-Name: 7b8d17dd840f64dac9a018a4547a97de799e94ab --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/pcache1.c | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 67a5b345e2..5ea50eda2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Compiler-warning\sfixes\sin\sthe\ssqldiff.exe\sutility. -D 2015-08-20T23:33:09.816 +C Fix\stypo\sin\scomment.\s\sNo\schanges\sto\scode. +D 2015-08-21T12:37:49.414 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4f663b6b4954b9b1eb0e6f08387688a93b57542d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -328,7 +328,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 F src/parse.y ad9af8552f6f340bd646577ca63356a6f82b6a7e F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 -F src/pcache1.c d08939800abf3031bd0affd5a13fbc4d7ba3fb68 +F src/pcache1.c a3fe31b17e841ec70beee72a2c960e9c787a8857 F src/pragma.c 669bc0fdb3fb5554e18330e8dd9743319bac16f4 F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 @@ -1376,8 +1376,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bc577fe6cbbe5385d81d6fa0f3c34bb1c833f0d6 072279d458fbb74a812a9ee723041d6b7c662a88 -R 945241c39c0e2b8c88fe2d95b8e90177 -T +closed 072279d458fbb74a812a9ee723041d6b7c662a88 +P 64d13339d44d1b7ec6768a33421f2138cb7872d8 +R 260ec5cb722f012aed82a5e5217a1778 U drh -Z 016c7dc0deb1c398804e42d942b63fbc +Z 32b72495e2d0c8597a1f587799e93112 diff --git a/manifest.uuid b/manifest.uuid index cb7f7ee87d..e438ad073b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -64d13339d44d1b7ec6768a33421f2138cb7872d8 \ No newline at end of file +7b8d17dd840f64dac9a018a4547a97de799e94ab \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index 187f09f592..bb01d66c4f 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -65,7 +65,7 @@ ** that is allocated when the page cache is created. The size of the local ** bulk allocation can be adjusted using ** -** sqlite3_config(SQLITE_CONFIG_PCACHE, 0, 0, N). +** sqlite3_config(SQLITE_CONFIG_PAGECACHE, 0, 0, N). ** ** If N is positive, then N pages worth of memory are allocated using a single ** sqlite3Malloc() call and that memory is used for the first N pages allocated. From 1f2fc281348e21475f093911d421545cb2aa1976 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Aug 2015 17:14:48 +0000 Subject: [PATCH 17/54] Fix a corner-case bug in table-valued functions. Update the generate_series() virtual table to increase the performance estimate penalty for being underspecified. FossilOrigin-Name: 552bc9cb88bbe54b4cf5fdf66d1217e7a2047110 --- ext/misc/series.c | 12 +++++++----- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/whereexpr.c | 1 + test/tabfunc01.test | 4 ++++ 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ext/misc/series.c b/ext/misc/series.c index 21f95ccb74..2ae312effa 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -334,19 +334,21 @@ static int seriesBestIndex( pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg; pIdxInfo->aConstraintUsage[stepIdx].omit = 1; } - if( pIdxInfo->nOrderBy==1 ){ - if( pIdxInfo->aOrderBy[0].desc ) idxNum |= 8; - pIdxInfo->orderByConsumed = 1; - } if( (idxNum & 3)==3 ){ /* Both start= and stop= boundaries are available. This is the ** the preferred case */ pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedRows = 1000; + if( pIdxInfo->nOrderBy==1 ){ + if( pIdxInfo->aOrderBy[0].desc ) idxNum |= 8; + pIdxInfo->orderByConsumed = 1; + } }else{ /* If either boundary is missing, we have to generate a huge span ** of numbers. Make this case very expensive so that the query ** planner will work hard to avoid it. */ - pIdxInfo->estimatedCost = (double)2000000000; + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; } pIdxInfo->idxNum = idxNum; return SQLITE_OK; diff --git a/manifest b/manifest index 5ea50eda2b..b33c9469b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\scomment.\s\sNo\schanges\sto\scode. -D 2015-08-21T12:37:49.414 +C Fix\sa\scorner-case\sbug\sin\stable-valued\sfunctions.\s\sUpdate\sthe\sgenerate_series()\nvirtual\stable\sto\sincrease\sthe\sperformance\sestimate\spenalty\sfor\sbeing\nunderspecified. +D 2015-08-21T17:14:48.307 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4f663b6b4954b9b1eb0e6f08387688a93b57542d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -196,7 +196,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a -F ext/misc/series.c 6f94daf590d0668187631dee2a4d7e1d8f3095c3 +F ext/misc/series.c b8fb7befd85b3a9b4a10e701b30b2b79ca92b6d4 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 86998fb73aefb7b5dc346ba8a58912f312da4996 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -416,7 +416,7 @@ F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c 66518a14a1238611aa0744d6980b6b7f544f4816 F src/whereInt.h 880a8599226ac1c00203490d934f3ed79b292572 F src/wherecode.c 69f19535a6de0cceb10e16b31a3a03463e31bc24 -F src/whereexpr.c f9dbd159127452150c92b558e184827ecb8f9229 +F src/whereexpr.c 1a308d1ee5144890d21ea9cf70d49bc96a83432b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1031,7 +1031,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/tabfunc01.test d556af2def6af10b0a759b2f8a8f41135c2b634e +F test/tabfunc01.test a12eba3f48a03a6626f985734ecc28132381fa9b F test/table.test 33bf0d1fd07f304582695184b8e6feb017303816 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -1376,7 +1376,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 64d13339d44d1b7ec6768a33421f2138cb7872d8 -R 260ec5cb722f012aed82a5e5217a1778 +P 7b8d17dd840f64dac9a018a4547a97de799e94ab +R 9cda90020c632a4b5578f3716c43ea3f U drh -Z 32b72495e2d0c8597a1f587799e93112 +Z 56572d82de750944343824c6ac2ac974 diff --git a/manifest.uuid b/manifest.uuid index e438ad073b..ae5ce62080 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b8d17dd840f64dac9a018a4547a97de799e94ab \ No newline at end of file +552bc9cb88bbe54b4cf5fdf66d1217e7a2047110 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index d6f94b3e1a..88eb5b70aa 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1281,6 +1281,7 @@ void sqlite3WhereTabFuncArgs( if( pColRef==0 ) return; pColRef->iTable = pItem->iCursor; pColRef->iColumn = k++; + pColRef->pTab = pTab; pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 39264d2f05..30a40e3138 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -65,5 +65,9 @@ do_execsql_test tabfunc01-2.2 { SELECT * FROM generate_series() LIMIT 5; } {0 1 2 3 4} +do_execsql_test tabfunc01-3.1 { + SELECT DISTINCT value FROM generate_series(1,x), t1 ORDER BY 1; +} {1 2 3} + finish_test From 505ad2ca41b1b7026ab74680d11179824d69c653 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Aug 2015 17:33:11 +0000 Subject: [PATCH 18/54] Add the json_tree() virtual table. FossilOrigin-Name: 08c36e45f0d3a7b89caf823652d7543b76ac802a --- ext/misc/json.c | 348 +++++++++++++++++++++++++++++++++++------------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 263 insertions(+), 99 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 77bc6d6794..26cde61056 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -34,7 +34,7 @@ typedef unsigned int u32; typedef unsigned char u8; /* Objects */ -typedef struct Json Json; +typedef struct JsonString JsonString; typedef struct JsonNode JsonNode; typedef struct JsonParse JsonParse; @@ -42,7 +42,7 @@ typedef struct JsonParse JsonParse; ** under construction. Really, this is a generic string accumulator ** that can be and is used to create strings other than JSON. */ -struct Json { +struct JsonString { sqlite3_context *pCtx; /* Function context - put error messages here */ char *zBuf; /* Append JSON content here */ u64 nAlloc; /* Bytes of storage available in zBuf[] */ @@ -89,6 +89,7 @@ struct JsonNode { union { const char *zJContent; /* Content for INT, REAL, and STRING */ u32 iAppend; /* More terms for ARRAY and OBJECT */ + u32 iKey; /* Key for ARRAY objects in json_tree() */ } u; }; @@ -99,44 +100,36 @@ struct JsonParse { u32 nAlloc; /* Number of slots of aNode[] allocated */ JsonNode *aNode; /* Array of nodes containing the parse */ const char *zJson; /* Original JSON string */ + u32 *aUp; /* Index of parent of each node */ u8 oom; /* Set to true if out of memory */ }; -/* -** Return the number of consecutive JsonNode slots need to represent -** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and -** OBJECT types, the number might be larger. -** -** Appended elements are not counted. The value returned is the number -** by which the JsonNode counter should increment in order to go to the -** next peer value. -*/ -static u32 jsonSize(JsonNode *pNode){ - return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; -} +/************************************************************************** +** Utility routines for dealing with JsonString objects +**************************************************************************/ -/* Set the Json object to an empty string +/* Set the JsonString object to an empty string */ -static void jsonZero(Json *p){ +static void jsonZero(JsonString *p){ p->zBuf = p->zSpace; p->nAlloc = sizeof(p->zSpace); p->nUsed = 0; p->bStatic = 1; } -/* Initialize the Json object +/* Initialize the JsonString object */ -static void jsonInit(Json *p, sqlite3_context *pCtx){ +static void jsonInit(JsonString *p, sqlite3_context *pCtx){ p->pCtx = pCtx; p->bErr = 0; jsonZero(p); } -/* Free all allocated memory and reset the Json object back to its +/* Free all allocated memory and reset the JsonString object back to its ** initial state. */ -static void jsonReset(Json *p){ +static void jsonReset(JsonString *p){ if( !p->bStatic ) sqlite3_free(p->zBuf); jsonZero(p); } @@ -144,7 +137,7 @@ static void jsonReset(Json *p){ /* Report an out-of-memory (OOM) condition */ -static void jsonOom(Json *p){ +static void jsonOom(JsonString *p){ if( !p->bErr ){ p->bErr = 1; sqlite3_result_error_nomem(p->pCtx); @@ -155,7 +148,7 @@ static void jsonOom(Json *p){ /* Enlarge pJson->zBuf so that it can hold at least N more bytes. ** Return zero on success. Return non-zero on an OOM error */ -static int jsonGrow(Json *p, u32 N){ +static int jsonGrow(JsonString *p, u32 N){ u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ @@ -180,9 +173,9 @@ static int jsonGrow(Json *p, u32 N){ return SQLITE_OK; } -/* Append N bytes from zIn onto the end of the Json string. +/* Append N bytes from zIn onto the end of the JsonString string. */ -static void jsonAppendRaw(Json *p, const char *zIn, u32 N){ +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; @@ -191,14 +184,14 @@ static void jsonAppendRaw(Json *p, const char *zIn, u32 N){ #ifdef SQLITE_DEBUG /* Append the zero-terminated string zIn */ -static void jsonAppend(Json *p, const char *zIn){ +static void jsonAppend(JsonString *p, const char *zIn){ jsonAppendRaw(p, zIn, (u32)strlen(zIn)); } #endif /* Append a single character */ -static void jsonAppendChar(Json *p, char c){ +static void jsonAppendChar(JsonString *p, char c){ if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; p->zBuf[p->nUsed++] = c; } @@ -206,19 +199,19 @@ static void jsonAppendChar(Json *p, char c){ /* Append a comma separator to the output buffer, if the previous ** character is not '[' or '{'. */ -static void jsonAppendSeparator(Json *p){ +static void jsonAppendSeparator(JsonString *p){ char c; if( p->nUsed==0 ) return; c = p->zBuf[p->nUsed-1]; if( c!='[' && c!='{' ) jsonAppendChar(p, ','); } -/* Append the N-byte string in zIn to the end of the Json string +/* Append the N-byte string in zIn to the end of the JsonString string ** under construction. Enclose the string in "..." and escape ** any double-quotes or backslash characters contained within the ** string. */ -static void jsonAppendString(Json *p, const char *zIn, u32 N){ +static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ u32 i; if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; p->zBuf[p->nUsed++] = '"'; @@ -238,7 +231,7 @@ static void jsonAppendString(Json *p, const char *zIn, u32 N){ ** construction. */ static void jsonAppendValue( - Json *p, /* Append to this JSON string */ + JsonString *p, /* Append to this JSON string */ sqlite3_value *pValue /* Value to append */ ){ switch( sqlite3_value_type(pValue) ){ @@ -273,7 +266,7 @@ static void jsonAppendValue( /* Make the JSON in p the result of the SQL function. */ -static void jsonResult(Json *p){ +static void jsonResult(JsonString *p){ if( p->bErr==0 ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, @@ -283,6 +276,36 @@ static void jsonResult(Json *p){ assert( p->bStatic ); } +/************************************************************************** +** Utility routines for dealing with JsonNode and JsonParse objects +**************************************************************************/ + +/* +** Return the number of consecutive JsonNode slots need to represent +** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and +** OBJECT types, the number might be larger. +** +** Appended elements are not counted. The value returned is the number +** by which the JsonNode counter should increment in order to go to the +** next peer value. +*/ +static u32 jsonNodeSize(JsonNode *pNode){ + return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; +} + +/* +** Reclaim all memory allocated by a JsonParse object. But do not +** delete the JsonParse object itself. +*/ +static void jsonParseReset(JsonParse *pParse){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + pParse->nNode = 0; + pParse->nAlloc = 0; + sqlite3_free(pParse->aUp); + pParse->aUp = 0; +} + /* ** Convert the JsonNode pNode into a pure JSON string and ** append to pOut. Subsubstructure is also included. Return @@ -290,7 +313,7 @@ static void jsonResult(Json *p){ */ static void jsonRenderNode( JsonNode *pNode, /* The node to render */ - Json *pOut, /* Write JSON here */ + JsonString *pOut, /* Write JSON here */ sqlite3_value **aReplace /* Replacement values */ ){ switch( pNode->eType ){ @@ -332,7 +355,7 @@ static void jsonRenderNode( jsonAppendSeparator(pOut); jsonRenderNode(&pNode[j], pOut, aReplace); } - j += jsonSize(&pNode[j]); + j += jsonNodeSize(&pNode[j]); } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; pNode = &pNode[pNode->u.iAppend]; @@ -356,7 +379,7 @@ static void jsonRenderNode( jsonRenderNode(&pNode[j+1], pOut, aReplace); } } - j += 1 + jsonSize(&pNode[j+1]); + j += 1 + jsonNodeSize(&pNode[j+1]); } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; pNode = &pNode[pNode->u.iAppend]; @@ -478,7 +501,7 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - Json s; + JsonString s; jsonInit(&s, pCtx); jsonRenderNode(pNode, &s, aReplace); jsonResult(&s); @@ -676,15 +699,51 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ if( zJson[i] ) i = -1; } if( i<0 ){ - sqlite3_free(pParse->aNode); - pParse->aNode = 0; - pParse->nNode = 0; - pParse->nAlloc = 0; + jsonParseReset(pParse); return 1; } return 0; } +/* Mark node i of pParse as being a child of iParent. Call recursively +** to fill in all the descendants of node i. +*/ +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ + JsonNode *pNode = &pParse->aNode[i]; + u32 j; + pParse->aUp[i] = iParent; + switch( pNode->eType ){ + case JSON_ARRAY: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ + jsonParseFillInParentage(pParse, i+j, i); + } + break; + } + case JSON_OBJECT: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ + pParse->aUp[i+j] = i; + jsonParseFillInParentage(pParse, i+j+1, i); + } + break; + } + default: { + break; + } + } +} + +/* +** Compute the parentage of all nodes in a completed parse. +*/ +static int jsonParseFindParents(JsonParse *pParse){ + u32 *aUp; + assert( pParse->aUp==0 ); + aUp = pParse->aUp = sqlite3_malloc( sizeof(u32)*pParse->nNode ); + if( aUp==0 ) return SQLITE_NOMEM; + jsonParseFillInParentage(pParse, 0, 0); + return SQLITE_OK; +} + /* forward declaration */ static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*); @@ -730,7 +789,7 @@ static JsonNode *jsonLookup( return jsonLookup(pParse, iRoot+j+1, &zPath[i], pApnd); } j++; - j += jsonSize(&pRoot[j]); + j += jsonNodeSize(&pRoot[j]); } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; iRoot += pRoot->u.iAppend; @@ -759,7 +818,7 @@ static JsonNode *jsonLookup( j = 1; for(;;){ while( i>0 && j<=pRoot->n ){ - j += jsonSize(&pRoot[j]); + j += jsonNodeSize(&pRoot[j]); i--; } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; @@ -820,8 +879,8 @@ static void jsonParseFunc( int argc, sqlite3_value **argv ){ - Json s; /* Output string - not real JSON */ - JsonParse x; /* The parse */ + JsonString s; /* Output string - not real JSON */ + JsonParse x; /* The parse */ u32 i; char zBuf[100]; @@ -838,7 +897,7 @@ static void jsonParseFunc( jsonAppendRaw(&s, "\n", 1); } } - sqlite3_free(x.aNode); + jsonParseReset(&x); jsonResult(&s); } @@ -853,7 +912,7 @@ static void jsonTest1Func( JsonParse x; /* The parse */ if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; jsonReturn(x.aNode, context, 0); - sqlite3_free(x.aNode); + jsonParseReset(&x); } /* @@ -868,7 +927,7 @@ static void jsonNodeCountFunc( JsonParse x; /* The parse */ if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; sqlite3_result_int64(context, x.nNode); - sqlite3_free(x.aNode); + jsonParseReset(&x); } #endif /* SQLITE_DEBUG */ @@ -887,7 +946,7 @@ static void jsonArrayFunc( sqlite3_value **argv ){ int i; - Json jx; + JsonString jx; jsonInit(&jx, context); jsonAppendChar(&jx, '['); @@ -932,11 +991,11 @@ static void jsonArrayLengthFunc( if( pNode->eType==JSON_ARRAY ){ assert( (pNode->jnFlags & JNODE_APPEND)==0 ); for(i=1; i<=pNode->n; n++){ - i += jsonSize(&pNode[i]); + i += jsonNodeSize(&pNode[i]); } } } - sqlite3_free(x.aNode); + jsonParseReset(&x); } sqlite3_result_int64(context, n); } @@ -965,7 +1024,7 @@ static void jsonExtractFunc( if( pNode ){ jsonReturn(pNode, context, 0); } - sqlite3_free(x.aNode); + jsonParseReset(&x); } /* @@ -979,7 +1038,7 @@ static void jsonObjectFunc( sqlite3_value **argv ){ int i; - Json jx; + JsonString jx; const char *z; u32 n; @@ -1039,7 +1098,7 @@ static void jsonRemoveFunc( jsonReturn(x.aNode, context, 0); } } - sqlite3_free(x.aNode); + jsonParseReset(&x); } /* @@ -1082,8 +1141,9 @@ static void jsonReplaceFunc( jsonReturn(x.aNode, context, argv); } } - sqlite3_free(x.aNode); + jsonParseReset(&x); } + /* ** json_set(JSON, PATH, VALUE, ...) ** @@ -1133,7 +1193,7 @@ static void jsonSetFunc( jsonReturn(x.aNode, context, argv); } } - sqlite3_free(x.aNode); + jsonParseReset(&x); } /* @@ -1165,7 +1225,7 @@ static void jsonTypeFunc( if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); sqlite3_result_text(context, jsonType[pNode->eType], -1, SQLITE_STATIC); } - sqlite3_free(x.aNode); + jsonParseReset(&x); } /**************************************************************************** @@ -1174,13 +1234,14 @@ static void jsonTypeFunc( typedef struct JsonEachCursor JsonEachCursor; struct JsonEachCursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - u32 iRowid; /* The rowid */ - u32 i; /* Index in sParse.aNode[] of current row */ - u32 iEnd; /* EOF when i equals or exceeds this value */ - u8 eType; /* Type of top-level element */ - char *zJson; /* Input json */ - char *zPath; /* Path by which to filter zJson */ - JsonParse sParse; /* The input json */ + u32 iRowid; /* The rowid */ + u32 i; /* Index in sParse.aNode[] of current row */ + u32 iEnd; /* EOF when i equals or exceeds this value */ + u8 eType; /* Type of top-level element */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ + char *zJson; /* Input JSON */ + char *zPath; /* Path by which to filter zJson */ + JsonParse sParse; /* Parse of the input JSON */ }; /* Constructor for the json_each virtual table */ @@ -1192,18 +1253,26 @@ static int jsonEachConnect( char **pzErr ){ sqlite3_vtab *pNew; - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); - if( pNew==0 ) return SQLITE_NOMEM; + int rc; /* Column numbers */ #define JEACH_KEY 0 #define JEACH_VALUE 1 -#define JEACH_JSON 2 -#define JEACH_PATH 3 +#define JEACH_TYPE 2 +#define JEACH_ATOM 3 +#define JEACH_ID 4 +#define JEACH_PARENT 5 +#define JEACH_JSON 6 +#define JEACH_PATH 7 - sqlite3_declare_vtab(db, "CREATE TABLE x(key,value,json hidden,path hidden)"); - memset(pNew, 0, sizeof(*pNew)); - return SQLITE_OK; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(key,value,type,atom,id,parent,json hidden,path hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; } /* destructor for json_each virtual table */ @@ -1212,8 +1281,8 @@ static int jsonEachDisconnect(sqlite3_vtab *pVtab){ return SQLITE_OK; } -/* constructor for a JsonEachCursor object. */ -static int jsonEachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ +/* constructor for a JsonEachCursor object for json_each(). */ +static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ JsonEachCursor *pCur; pCur = sqlite3_malloc( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; @@ -1222,17 +1291,26 @@ static int jsonEachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } +/* constructor for a JsonEachCursor object for json_tree(). */ +static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + int rc = jsonEachOpenEach(p, ppCursor); + if( rc==SQLITE_OK ){ + JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; + pCur->bRecursive = 1; + } + return rc; +} + /* Reset a JsonEachCursor back to its original state. Free any memory ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ sqlite3_free(p->zJson); sqlite3_free(p->zPath); - sqlite3_free(p->sParse.aNode); + jsonParseReset(&p->sParse); p->iRowid = 0; p->i = 0; p->iEnd = 0; p->eType = 0; - memset(&p->sParse, 0, sizeof(p->sParse)); p->zJson = 0; p->zPath = 0; } @@ -1252,18 +1330,39 @@ static int jsonEachEof(sqlite3_vtab_cursor *cur){ return p->i >= p->iEnd; } -/* Advance the cursor to the next top-level element of the current -** JSON string */ -static int jsonEachNext(sqlite3_vtab_cursor *cur){ +/* Advance the cursor to the next element for json_tree() */ +static int jsonEachNextTree(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + if( p->i==0 ){ + p->i = 1; + }else if( p->sParse.aNode[p->sParse.aUp[p->i]].eType==JSON_OBJECT ){ + p->i += 2; + }else{ + p->i++; + } + p->iRowid++; + if( p->isParse.nNode ){ + JsonNode *pUp = &p->sParse.aNode[p->sParse.aUp[p->i]]; + p->eType = pUp->eType; + if( pUp->eType==JSON_ARRAY ) pUp->u.iKey++; + if( p->sParse.aNode[p->i].eType==JSON_ARRAY ){ + p->sParse.aNode[p->i].u.iKey = 0; + } + } + return SQLITE_OK; +} + +/* Advance the cursor to the next element for json_each() */ +static int jsonEachNextEach(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; switch( p->eType ){ case JSON_ARRAY: { - p->i += jsonSize(&p->sParse.aNode[p->i]); + p->i += jsonNodeSize(&p->sParse.aNode[p->i]); p->iRowid++; break; } case JSON_OBJECT: { - p->i += 1 + jsonSize(&p->sParse.aNode[p->i+1]); + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); p->iRowid++; break; } @@ -1282,20 +1381,46 @@ static int jsonEachColumn( int i /* Which column to return */ ){ JsonEachCursor *p = (JsonEachCursor*)cur; + JsonNode *pThis = &p->sParse.aNode[p->i]; switch( i ){ case JEACH_KEY: { if( p->eType==JSON_OBJECT ){ - jsonReturn(&p->sParse.aNode[p->i], ctx, 0); - }else{ - sqlite3_result_int64(ctx, p->iRowid); + jsonReturn(pThis, ctx, 0); + }else if( p->eType==JSON_ARRAY ){ + u32 iKey; + if( p->bRecursive ){ + if( p->iRowid==0 ) break; + iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey - 1; + }else{ + iKey = p->iRowid; + } + sqlite3_result_int64(ctx, iKey); } break; } case JEACH_VALUE: { - if( p->eType==JSON_OBJECT ){ - jsonReturn(&p->sParse.aNode[p->i+1], ctx, 0); - }else{ - jsonReturn(&p->sParse.aNode[p->i], ctx, 0); + if( p->eType==JSON_OBJECT ) pThis++; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_TYPE: { + if( p->eType==JSON_OBJECT ) pThis++; + sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); + break; + } + case JEACH_ATOM: { + if( p->eType==JSON_OBJECT ) pThis++; + if( pThis->eType>=JSON_ARRAY ) break; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_ID: { + sqlite3_result_int64(ctx, p->i + (p->eType==JSON_OBJECT)); + break; + } + case JEACH_PARENT: { + if( p->i>0 && p->bRecursive ){ + sqlite3_result_int64(ctx, p->sParse.aUp[p->i]); } break; } @@ -1306,6 +1431,7 @@ static int jsonEachColumn( break; } default: { + assert( i==JEACH_JSON ); sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); break; } @@ -1345,9 +1471,9 @@ static int jsonEachBestIndex( } if( jsonIdx<0 ){ pIdxInfo->idxNum = 0; - pIdxInfo->estimatedCost = (double)2000000000; + pIdxInfo->estimatedCost = 1e99; }else{ - pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedCost = 1.0; pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1; pIdxInfo->aConstraintUsage[jsonIdx].omit = 1; if( pathIdx<0 ){ @@ -1384,7 +1510,9 @@ static int jsonEachFilter( p->zJson = sqlite3_malloc( n+1 ); if( p->zJson==0 ) return SQLITE_NOMEM; memcpy(p->zJson, z, n+1); - if( jsonParse(&p->sParse, p->zJson) ){ + if( jsonParse(&p->sParse, p->zJson) + || (p->bRecursive && jsonParseFindParents(&p->sParse)) + ){ jsonEachCursorReset(p); }else{ JsonNode *pNode; @@ -1421,10 +1549,10 @@ static sqlite3_module jsonEachModule = { jsonEachBestIndex, /* xBestIndex */ jsonEachDisconnect, /* xDisconnect */ 0, /* xDestroy */ - jsonEachOpen, /* xOpen - open a cursor */ + jsonEachOpenEach, /* xOpen - open a cursor */ jsonEachClose, /* xClose - close a cursor */ jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNext, /* xNext - advance a cursor */ + jsonEachNextEach, /* xNext - advance a cursor */ jsonEachEof, /* xEof - check for end of scan */ jsonEachColumn, /* xColumn - read data */ jsonEachRowid, /* xRowid - read data */ @@ -1437,6 +1565,35 @@ static sqlite3_module jsonEachModule = { 0, /* xRename */ }; +/* The methods of the json_tree virtual table. */ +static sqlite3_module jsonTreeModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenTree, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNextTree, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +/**************************************************************************** +** The following routine is the only publically visible identifier in this +** file. Call the following routine in order to register the various SQL +** functions and the virtual table implemented by this file. +****************************************************************************/ #ifdef _WIN32 __declspec(dllexport) @@ -1473,6 +1630,13 @@ int sqlite3_json_init( { "json_nodecount", 1, 0, jsonNodeCountFunc }, #endif }; + static const struct { + const char *zName; + sqlite3_module *pModule; + } aMod[] = { + { "json_each", &jsonEachModule }, + { "json_tree", &jsonTreeModule }, + }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ for(i=0; i Date: Fri, 21 Aug 2015 17:39:35 +0000 Subject: [PATCH 19/54] Reserve the SQLITE_IOERR_VNODE error code name. FossilOrigin-Name: 53b593fcae178f2e08f758ae6fd100869f771bfd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b33c9469b9..f6550d05fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scorner-case\sbug\sin\stable-valued\sfunctions.\s\sUpdate\sthe\sgenerate_series()\nvirtual\stable\sto\sincrease\sthe\sperformance\sestimate\spenalty\sfor\sbeing\nunderspecified. -D 2015-08-21T17:14:48.307 +C Reserve\sthe\sSQLITE_IOERR_VNODE\serror\scode\sname. +D 2015-08-21T17:39:35.092 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4f663b6b4954b9b1eb0e6f08387688a93b57542d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -338,7 +338,7 @@ F src/resolve.c 7a67cd2aebc9a9eeecd1d104eb6a9237388eb452 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c c46de38c1b66355f02a839bb72eb13f277e6d19c F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 -F src/sqlite.h.in 447ead0a6b3293206f04a0896553955d07cfb4b9 +F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h a0b948ebc89bac13941254641326a6aa248c2cc4 F src/sqliteInt.h 89e68539d645db597366a91411468b51e73c21a0 @@ -1376,7 +1376,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7b8d17dd840f64dac9a018a4547a97de799e94ab -R 9cda90020c632a4b5578f3716c43ea3f +P 552bc9cb88bbe54b4cf5fdf66d1217e7a2047110 +R 7e195d8a93b6662d59cab96efed45d96 U drh -Z 56572d82de750944343824c6ac2ac974 +Z cc2b20b9f86d6ace20a4a1a638a837d0 diff --git a/manifest.uuid b/manifest.uuid index ae5ce62080..847125d408 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -552bc9cb88bbe54b4cf5fdf66d1217e7a2047110 \ No newline at end of file +53b593fcae178f2e08f758ae6fd100869f771bfd \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 99fb4eb588..a7d6318e5b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -477,6 +477,7 @@ int sqlite3_exec( #define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) #define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) +#define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) From cfe05d882f2fdb6497c47c9e382a2ba6a3030f2d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Aug 2015 19:53:06 +0000 Subject: [PATCH 20/54] Add a missing #define for sqlite3_vsnprintf to sqlite3ext.h. FossilOrigin-Name: da3c9df09c46564353218d0163e378b880a3ce62 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite3ext.h | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f6550d05fe..667104cb07 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reserve\sthe\sSQLITE_IOERR_VNODE\serror\scode\sname. -D 2015-08-21T17:39:35.092 +C Add\sa\smissing\s#define\sfor\ssqlite3_vsnprintf\sto\ssqlite3ext.h. +D 2015-08-21T19:53:06.112 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 4f663b6b4954b9b1eb0e6f08387688a93b57542d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -340,7 +340,7 @@ F src/select.c c46de38c1b66355f02a839bb72eb13f277e6d19c F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad -F src/sqlite3ext.h a0b948ebc89bac13941254641326a6aa248c2cc4 +F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 F src/sqliteInt.h 89e68539d645db597366a91411468b51e73c21a0 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 @@ -1376,7 +1376,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 552bc9cb88bbe54b4cf5fdf66d1217e7a2047110 -R 7e195d8a93b6662d59cab96efed45d96 +P 53b593fcae178f2e08f758ae6fd100869f771bfd +R ebc4801d5f618bfa6944c04f2d984190 U drh -Z cc2b20b9f86d6ace20a4a1a638a837d0 +Z 0c4320af06ae345cfafae29306ed4bea diff --git a/manifest.uuid b/manifest.uuid index 847125d408..646c99b26a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -53b593fcae178f2e08f758ae6fd100869f771bfd \ No newline at end of file +da3c9df09c46564353218d0163e378b880a3ce62 \ No newline at end of file diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 48a5bf744b..1525327f49 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -412,6 +412,7 @@ struct sqlite3_api_routines { #define sqlite3_value_text16le sqlite3_api->value_text16le #define sqlite3_value_type sqlite3_api->value_type #define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_vsnprintf sqlite3_api->vsnprintf #define sqlite3_overload_function sqlite3_api->overload_function #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 From 4af352d47edaa53f860805157b00fc6cfcdaa155 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Aug 2015 20:02:48 +0000 Subject: [PATCH 21/54] Add the fullkey column to both json_each() and json_tree(). FossilOrigin-Name: 15dd99431e9ddd0fbdbb8dcc921687b0c6d26a29 --- ext/misc/json.c | 158 ++++++++++++++++++++++++++++++++++-------------- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 120 insertions(+), 52 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 26cde61056..9d9bf63896 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -27,6 +27,7 @@ SQLITE_EXTENSION_INIT1 #include #include #include +#include /* Unsigned integer types */ typedef sqlite3_uint64 u64; @@ -181,6 +182,17 @@ static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ p->nUsed += N; } +/* Append formatted text (not to exceed N bytes) to the JsonString. +*/ +static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ + va_list ap; + if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + va_start(ap, zFormat); + sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); + va_end(ap); + p->nUsed += (int)strlen(p->zBuf+p->nUsed); +} + #ifdef SQLITE_DEBUG /* Append the zero-terminated string zIn */ @@ -1256,17 +1268,19 @@ static int jsonEachConnect( int rc; /* Column numbers */ -#define JEACH_KEY 0 -#define JEACH_VALUE 1 -#define JEACH_TYPE 2 -#define JEACH_ATOM 3 -#define JEACH_ID 4 -#define JEACH_PARENT 5 -#define JEACH_JSON 6 -#define JEACH_PATH 7 +#define JEACH_KEY 0 +#define JEACH_VALUE 1 +#define JEACH_TYPE 2 +#define JEACH_ATOM 3 +#define JEACH_ID 4 +#define JEACH_PARENT 5 +#define JEACH_FULLKEY 6 +#define JEACH_JSON 7 +#define JEACH_PATH 8 rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(key,value,type,atom,id,parent,json hidden,path hidden)"); + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey," + "json HIDDEN,path HIDDEN)"); if( rc==SQLITE_OK ){ pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; @@ -1331,47 +1345,71 @@ static int jsonEachEof(sqlite3_vtab_cursor *cur){ } /* Advance the cursor to the next element for json_tree() */ -static int jsonEachNextTree(sqlite3_vtab_cursor *cur){ +static int jsonEachNext(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; - if( p->i==0 ){ - p->i = 1; - }else if( p->sParse.aNode[p->sParse.aUp[p->i]].eType==JSON_OBJECT ){ - p->i += 2; + if( p->bRecursive ){ + if( p->i==0 ){ + p->i = 1; + }else if( p->sParse.aNode[p->sParse.aUp[p->i]].eType==JSON_OBJECT ){ + p->i += 2; + }else{ + p->i++; + } + p->iRowid++; + if( p->isParse.nNode ){ + JsonNode *pUp = &p->sParse.aNode[p->sParse.aUp[p->i]]; + p->eType = pUp->eType; + if( pUp->eType==JSON_ARRAY ) pUp->u.iKey++; + if( p->sParse.aNode[p->i].eType==JSON_ARRAY ){ + p->sParse.aNode[p->i].u.iKey = 0; + } + } }else{ - p->i++; - } - p->iRowid++; - if( p->isParse.nNode ){ - JsonNode *pUp = &p->sParse.aNode[p->sParse.aUp[p->i]]; - p->eType = pUp->eType; - if( pUp->eType==JSON_ARRAY ) pUp->u.iKey++; - if( p->sParse.aNode[p->i].eType==JSON_ARRAY ){ - p->sParse.aNode[p->i].u.iKey = 0; + switch( p->eType ){ + case JSON_ARRAY: { + p->i += jsonNodeSize(&p->sParse.aNode[p->i]); + p->iRowid++; + break; + } + case JSON_OBJECT: { + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); + p->iRowid++; + break; + } + default: { + p->i = p->iEnd; + break; + } } } return SQLITE_OK; } -/* Advance the cursor to the next element for json_each() */ -static int jsonEachNextEach(sqlite3_vtab_cursor *cur){ - JsonEachCursor *p = (JsonEachCursor*)cur; - switch( p->eType ){ - case JSON_ARRAY: { - p->i += jsonNodeSize(&p->sParse.aNode[p->i]); - p->iRowid++; - break; - } - case JSON_OBJECT: { - p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); - p->iRowid++; - break; - } - default: { - p->i = p->iEnd; - break; - } +/* Append the name of the path for element i to pStr +*/ +static void jsonEachComputePath( + JsonEachCursor *p, /* The cursor */ + JsonString *pStr, /* Write the path here */ + u32 i /* Path to this element */ +){ + JsonNode *pNode, *pUp; + u32 iUp; + if( i==0 ){ + jsonAppendChar(pStr, '$'); + return; + } + iUp = p->sParse.aUp[i]; + jsonEachComputePath(p, pStr, iUp); + pNode = &p->sParse.aNode[i]; + pUp = &p->sParse.aNode[iUp]; + if( pUp->eType==JSON_ARRAY ){ + jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); + }else{ + assert( pUp->eType==JSON_OBJECT ); + if( pNode->eType>=JSON_ARRAY ) pNode--; + assert( pNode->eType==JSON_STRING ); + jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); } - return SQLITE_OK; } /* Return the value of a column */ @@ -1424,9 +1462,38 @@ static int jsonEachColumn( } break; } + case JEACH_FULLKEY: { + JsonString x; + jsonInit(&x, ctx); + if( p->bRecursive ){ + jsonEachComputePath(p, &x, p->i); + }else{ + if( p->zPath ){ + jsonAppendRaw(&x, p->zPath, (int)strlen(p->zPath)); + }else{ + jsonAppendChar(&x, '$'); + } + if( p->eType==JSON_ARRAY ){ + jsonPrintf(30, &x, "[%d]", p->iRowid); + }else{ + jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); + } + } + jsonResult(&x); + break; + } case JEACH_PATH: { const char *zPath = p->zPath; - if( zPath==0 ) zPath = "$"; + if( zPath==0 ){ + if( p->bRecursive ){ + JsonString x; + jsonInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonResult(&x); + break; + } + zPath = "$"; + } sqlite3_result_text(ctx, zPath, -1, SQLITE_STATIC); break; } @@ -1517,6 +1584,7 @@ static int jsonEachFilter( }else{ JsonNode *pNode; if( idxNum==3 ){ + p->bRecursive = 0; n = sqlite3_value_bytes(argv[1]); p->zPath = sqlite3_malloc( n+1 ); if( p->zPath==0 ) return SQLITE_NOMEM; @@ -1552,7 +1620,7 @@ static sqlite3_module jsonEachModule = { jsonEachOpenEach, /* xOpen - open a cursor */ jsonEachClose, /* xClose - close a cursor */ jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNextEach, /* xNext - advance a cursor */ + jsonEachNext, /* xNext - advance a cursor */ jsonEachEof, /* xEof - check for end of scan */ jsonEachColumn, /* xColumn - read data */ jsonEachRowid, /* xRowid - read data */ @@ -1576,7 +1644,7 @@ static sqlite3_module jsonTreeModule = { jsonEachOpenTree, /* xOpen - open a cursor */ jsonEachClose, /* xClose - close a cursor */ jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNextTree, /* xNext - advance a cursor */ + jsonEachNext, /* xNext - advance a cursor */ jsonEachEof, /* xEof - check for end of scan */ jsonEachColumn, /* xColumn - read data */ jsonEachRowid, /* xRowid - read data */ diff --git a/manifest b/manifest index 6e096c9ed7..a20995d28b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sheader\sfile\sfixes\sfrom\strunk. -D 2015-08-21T19:56:45.243 +C Add\sthe\sfullkey\scolumn\sto\sboth\sjson_each()\sand\sjson_tree(). +D 2015-08-21T20:02:48.424 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 90f3097efb9a53f5fc59a4f8a08be07cf9f52c02 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 522d833e6c4ae55b345cf25776aab335323d2f59 +F ext/misc/json.c 92bb4e5fe2956564d23f933e2140ce8e086d988b F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 08c36e45f0d3a7b89caf823652d7543b76ac802a da3c9df09c46564353218d0163e378b880a3ce62 -R 7de9031da9783d58b8a952bc530228c5 +P 7c2713e98ffb5f0d96eb7de9514eab43f0712011 +R a1c5d56719dd38b21713ad204a82abd5 U drh -Z 5f4a9a3778df860327f346518d686e5d +Z 8567373ad141fc34581702b17b3fb646 diff --git a/manifest.uuid b/manifest.uuid index ded172cdf0..7f4f0b9760 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c2713e98ffb5f0d96eb7de9514eab43f0712011 \ No newline at end of file +15dd99431e9ddd0fbdbb8dcc921687b0c6d26a29 \ No newline at end of file From f28ed795dbda2cba597368fc4c85b3ee487bf531 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Aug 2015 20:12:43 +0000 Subject: [PATCH 22/54] Change the name of the json loadable extension to "json1.c", in anticipation of having future major changes to the interface. FossilOrigin-Name: d0d4bec9e3d8829a2d488f2742f1650214fa716a --- ext/misc/{json.c => json1.c} | 0 manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) rename ext/misc/{json.c => json1.c} (100%) diff --git a/ext/misc/json.c b/ext/misc/json1.c similarity index 100% rename from ext/misc/json.c rename to ext/misc/json1.c diff --git a/manifest b/manifest index a20995d28b..25d78bc23f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sfullkey\scolumn\sto\sboth\sjson_each()\sand\sjson_tree(). -D 2015-08-21T20:02:48.424 +C Change\sthe\sname\sof\sthe\sjson\sloadable\sextension\sto\s"json1.c",\sin\santicipation\nof\shaving\sfuture\smajor\schanges\sto\sthe\sinterface. +D 2015-08-21T20:12:43.689 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 90f3097efb9a53f5fc59a4f8a08be07cf9f52c02 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 92bb4e5fe2956564d23f933e2140ce8e086d988b +F ext/misc/json1.c 92bb4e5fe2956564d23f933e2140ce8e086d988b w ext/misc/json.c F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7c2713e98ffb5f0d96eb7de9514eab43f0712011 -R a1c5d56719dd38b21713ad204a82abd5 +P 15dd99431e9ddd0fbdbb8dcc921687b0c6d26a29 +R 46781c36cc64ae92318031e0c6494ecf U drh -Z 8567373ad141fc34581702b17b3fb646 +Z cf5f4e21b0acc856e5deddedc237451f diff --git a/manifest.uuid b/manifest.uuid index 7f4f0b9760..02752a564b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15dd99431e9ddd0fbdbb8dcc921687b0c6d26a29 \ No newline at end of file +d0d4bec9e3d8829a2d488f2742f1650214fa716a \ No newline at end of file From 6fd5c1e0e859f08977defd48ea9385036c055d7c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Aug 2015 20:37:12 +0000 Subject: [PATCH 23/54] Fixes for compiler warnings and errors in the makefiles. Rename the one test script to json101.test. FossilOrigin-Name: 9ff6ccde5f26f18073587c320290570854ffc833 --- Makefile.in | 2 +- Makefile.msc | 2 +- ext/misc/json1.c | 46 +++++++++++++++++++++---------- main.mk | 2 +- manifest | 20 +++++++------- manifest.uuid | 2 +- test/{json1.test => json101.test} | 0 7 files changed, 45 insertions(+), 29 deletions(-) rename test/{json1.test => json101.test} (100%) diff --git a/Makefile.in b/Makefile.in index 38df655816..8f3d6f7dfe 100644 --- a/Makefile.in +++ b/Makefile.in @@ -417,7 +417,7 @@ TESTSRC += \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/json.c \ + $(TOP)/ext/misc/json1.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ diff --git a/Makefile.msc b/Makefile.msc index 5f64dfcc5e..441c499df8 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1083,7 +1083,7 @@ TESTEXT = \ $(TOP)\ext\fts5\fts5_tcl.c \ $(TOP)\ext\fts5\fts5_test_mi.c \ $(TOP)\ext\misc\ieee754.c \ - $(TOP)\ext\misc\json.c \ + $(TOP)\ext\misc\json1.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 9d9bf63896..66173482c4 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -29,6 +29,8 @@ SQLITE_EXTENSION_INIT1 #include #include +#define UNUSED_PARAM(X) (void)(X) + /* Unsigned integer types */ typedef sqlite3_uint64 u64; typedef unsigned int u32; @@ -159,7 +161,7 @@ static int jsonGrow(JsonString *p, u32 N){ jsonOom(p); return SQLITE_NOMEM; } - memcpy(zNew, p->zBuf, p->nUsed); + memcpy(zNew, p->zBuf, (size_t)p->nUsed); p->zBuf = zNew; p->bStatic = 0; }else{ @@ -578,7 +580,6 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c=='{' ){ /* Parse object */ iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - if( iThis<0 ) return -1; for(j=i+1;;j++){ while( isspace(pParse->zJson[j]) ){ j++; } x = jsonParseValue(pParse, j); @@ -605,7 +606,6 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ }else if( c=='[' ){ /* Parse array */ iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - if( iThis<0 ) return -1; for(j=i+1;;j++){ while( isspace(pParse->zJson[j]) ){ j++; } x = jsonParseValue(pParse, j); @@ -938,7 +938,7 @@ static void jsonNodeCountFunc( ){ JsonParse x; /* The parse */ if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; - sqlite3_result_int64(context, x.nNode); + sqlite3_result_int64(context, (sqlite3_int64)x.nNode); jsonParseReset(&x); } #endif /* SQLITE_DEBUG */ @@ -1099,7 +1099,7 @@ static void jsonRemoveFunc( if( argc<1 ) return; if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ - for(i=1; iiRowid; } - sqlite3_result_int64(ctx, iKey); + sqlite3_result_int64(ctx, (sqlite3_int64)iKey); } break; } @@ -1453,12 +1459,12 @@ static int jsonEachColumn( break; } case JEACH_ID: { - sqlite3_result_int64(ctx, p->i + (p->eType==JSON_OBJECT)); + sqlite3_result_int64(ctx, (sqlite3_int64)p->i + (p->eType==JSON_OBJECT)); break; } case JEACH_PARENT: { if( p->i>0 && p->bRecursive ){ - sqlite3_result_int64(ctx, p->sParse.aUp[p->i]); + sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); } break; } @@ -1526,6 +1532,8 @@ static int jsonEachBestIndex( int jsonIdx = -1; int pathIdx = -1; const struct sqlite3_index_constraint *pConstraint; + + UNUSED_PARAM(tab); pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ if( pConstraint->usable==0 ) continue; @@ -1565,6 +1573,8 @@ static int jsonEachFilter( const char *zPath; sqlite3_int64 n; + UNUSED_PARAM(idxStr); + UNUSED_PARAM(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; z = (const char*)sqlite3_value_text(argv[0]); @@ -1574,9 +1584,9 @@ static int jsonEachFilter( if( zPath==0 || zPath[0]!='$' ) return SQLITE_OK; } n = sqlite3_value_bytes(argv[0]); - p->zJson = sqlite3_malloc( n+1 ); + p->zJson = sqlite3_malloc64( n+1 ); if( p->zJson==0 ) return SQLITE_NOMEM; - memcpy(p->zJson, z, n+1); + memcpy(p->zJson, z, (size_t)n+1); if( jsonParse(&p->sParse, p->zJson) || (p->bRecursive && jsonParseFindParents(&p->sParse)) ){ @@ -1586,9 +1596,9 @@ static int jsonEachFilter( if( idxNum==3 ){ p->bRecursive = 0; n = sqlite3_value_bytes(argv[1]); - p->zPath = sqlite3_malloc( n+1 ); + p->zPath = sqlite3_malloc64( n+1 ); if( p->zPath==0 ) return SQLITE_NOMEM; - memcpy(p->zPath, zPath, n+1); + memcpy(p->zPath, zPath, (size_t)n+1); pNode = jsonLookup(&p->sParse, 0, p->zPath+1, 0); if( pNode==0 ){ jsonEachCursorReset(p); @@ -1631,6 +1641,9 @@ static sqlite3_module jsonEachModule = { 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; /* The methods of the json_tree virtual table. */ @@ -1655,6 +1668,9 @@ static sqlite3_module jsonTreeModule = { 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; /**************************************************************************** @@ -1672,7 +1688,7 @@ int sqlite3_json_init( const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; - int i; + unsigned int i; static const struct { const char *zName; int nArg; diff --git a/main.mk b/main.mk index e18370a3a1..9988fe2775 100644 --- a/main.mk +++ b/main.mk @@ -297,7 +297,7 @@ TESTSRC += \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/json.c \ + $(TOP)/ext/misc/json1.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ diff --git a/manifest b/manifest index 25d78bc23f..3c1e7613e4 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Change\sthe\sname\sof\sthe\sjson\sloadable\sextension\sto\s"json1.c",\sin\santicipation\nof\shaving\sfuture\smajor\schanges\sto\sthe\sinterface. -D 2015-08-21T20:12:43.689 +C Fixes\sfor\scompiler\swarnings\sand\serrors\sin\sthe\smakefiles.\s\sRename\sthe\none\stest\sscript\sto\sjson101.test. +D 2015-08-21T20:37:12.208 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 90f3097efb9a53f5fc59a4f8a08be07cf9f52c02 +F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc f6ed2ce438f711641dc63d762bb0d41aa659518f +F Makefile.msc 10af19cc089862481d49b347acd99c02635ddc49 F Makefile.vxworks e1b65dea203f054e71653415bd8f96dcaed47858 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION ccfc4d1576dbfdeece0a4372a2e6a2e37d3e7975 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 92bb4e5fe2956564d23f933e2140ce8e086d988b w ext/misc/json.c +F ext/misc/json1.c f83f02ec4d7cc18794bd7ac046f1bb5905805b44 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -258,7 +258,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a551a3b8a398e1e64fefbb14bc142aa9584238fc +F main.mk 04840e8277ab5159af16172eafd214dae7cffff5 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -810,7 +810,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json1.test 950ed4e8deb8ad4c10bd4fbc858eb54143de9867 +F test/json101.test 950ed4e8deb8ad4c10bd4fbc858eb54143de9867 w test/json1.test F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 15dd99431e9ddd0fbdbb8dcc921687b0c6d26a29 -R 46781c36cc64ae92318031e0c6494ecf +P d0d4bec9e3d8829a2d488f2742f1650214fa716a +R 0dbca2db2baae780df721a60ce2b8302 U drh -Z cf5f4e21b0acc856e5deddedc237451f +Z f736e312f15e55183b503d223fff9c6b diff --git a/manifest.uuid b/manifest.uuid index 02752a564b..a805312be8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0d4bec9e3d8829a2d488f2742f1650214fa716a \ No newline at end of file +9ff6ccde5f26f18073587c320290570854ffc833 \ No newline at end of file diff --git a/test/json1.test b/test/json101.test similarity index 100% rename from test/json1.test rename to test/json101.test From 38978dd4edf28da1b9c8f4003caecfafab31d206 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Aug 2015 01:32:29 +0000 Subject: [PATCH 24/54] Do not apply the WHERE-clause pushdown optimization to terms that originate in the ON or USING clause of a LEFT JOIN. Fix for ticket [6df18e949d3676290]. FossilOrigin-Name: 351bc22fa9b5a2e50da3583a882c5aa390bda19f --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/select.c | 4 ++++ test/join5.test | 23 +++++++++++++++++++++++ 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b1c03a8f7b..ab5e94e7a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextension\sfunctions\sfor\sprocessing\sJSON. -D 2015-08-21T20:43:32.515 +C Do\snot\sapply\sthe\sWHERE-clause\spushdown\soptimization\sto\sterms\sthat\soriginate\nin\sthe\sON\sor\sUSING\sclause\sof\sa\sLEFT\sJOIN.\s\sFix\sfor\sticket\n[6df18e949d3676290]. +D 2015-08-22T01:32:29.479 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -337,7 +337,7 @@ F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 7a67cd2aebc9a9eeecd1d104eb6a9237388eb452 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c c46de38c1b66355f02a839bb72eb13f277e6d19c +F src/select.c 24323faace224dce1e6b65276605de1db39d77a4 F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -802,7 +802,7 @@ F test/join.test f9d4a28dec81c6e9dc21b73518e024d73b5ebf57 F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 -F test/join5.test 5df23eba184f159ed9705a954957e765a10c141d +F test/join5.test 8a5c0be6f0c260a5c7177c3b8f07c7856141038a F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 69abc726c51b4a0409189f9a85191205297c0577 F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 @@ -1378,8 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P da3c9df09c46564353218d0163e378b880a3ce62 9ff6ccde5f26f18073587c320290570854ffc833 -R 0dbca2db2baae780df721a60ce2b8302 -T +closed 9ff6ccde5f26f18073587c320290570854ffc833 +P 178f9a352c6c9e15e809e1a47530c6592d18578d +R 3f8fa8b32979f76f9659e6c12a12d079 U drh -Z 2864a0dca372e608c85df3d0df7cbffa +Z fe48d12321db6aa71a1dc8fdceeea2c3 diff --git a/manifest.uuid b/manifest.uuid index 1298db00b3..661f57a8ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -178f9a352c6c9e15e809e1a47530c6592d18578d \ No newline at end of file +351bc22fa9b5a2e50da3583a882c5aa390bda19f \ No newline at end of file diff --git a/src/select.c b/src/select.c index acc4c88a57..e767111d8d 100644 --- a/src/select.c +++ b/src/select.c @@ -3757,6 +3757,9 @@ static int flattenSubquery( ** enforces this restriction since this routine does not have enough ** information to know.) ** +** (5) The WHERE clause expression originates in the ON or USING clause +** of a LEFT JOIN. +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -3779,6 +3782,7 @@ static int pushDownWhereTerms( nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); pWhere = pWhere->pLeft; } + if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ nChng++; while( pSubq ){ diff --git a/test/join5.test b/test/join5.test index b0b0df4f9d..543cd4d27f 100644 --- a/test/join5.test +++ b/test/join5.test @@ -161,4 +161,27 @@ do_execsql_test join5-3.3 { SELECT * FROM x1 LEFT JOIN x2 JOIN x3 WHERE x3.d = x2.b; } {} +# Ticket https://www.sqlite.org/src/tktview/c2a19d81652f40568c770c43 on +# 2015-08-20. LEFT JOIN and the push-down optimization. +# +do_execsql_test join6-4.1 { + SELECT * + FROM ( + SELECT 'apple' fruit + UNION ALL SELECT 'banana' + ) a + JOIN ( + SELECT 'apple' fruit + UNION ALL SELECT 'banana' + ) b ON a.fruit=b.fruit + LEFT JOIN ( + SELECT 1 isyellow + ) c ON b.fruit='banana'; +} {apple apple {} banana banana 1} +do_execsql_test join6-4.2 { + SELECT * + FROM (SELECT 'apple' fruit UNION ALL SELECT 'banana') + LEFT JOIN (SELECT 1) ON fruit='banana'; +} {apple {} banana 1} + finish_test From be9474ee05fd36fa5925847d9aa6cac9a06f9ad3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Aug 2015 03:05:54 +0000 Subject: [PATCH 25/54] Fix a couple instances of OOM handling in the json extension. FossilOrigin-Name: 213a6c5ccbcfe4495d45e3608e99a6398751aeed --- ext/misc/json1.c | 3 ++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 66173482c4..ea456bfeac 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -587,6 +587,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( x==(-2) && pParse->nNode==iThis+1 ) return j+1; return -1; } + if( pParse->oom ) return -1; if( pParse->aNode[pParse->nNode-1].eType!=JSON_STRING ) return -1; j = x; while( isspace(pParse->zJson[j]) ){ j++; } @@ -639,7 +640,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ j++; } jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]); - pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; return j+1; }else if( c=='n' && strncmp(pParse->zJson+i,"null",4)==0 diff --git a/manifest b/manifest index ab5e94e7a0..de7c994cbe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sapply\sthe\sWHERE-clause\spushdown\soptimization\sto\sterms\sthat\soriginate\nin\sthe\sON\sor\sUSING\sclause\sof\sa\sLEFT\sJOIN.\s\sFix\sfor\sticket\n[6df18e949d3676290]. -D 2015-08-22T01:32:29.479 +C Fix\sa\scouple\sinstances\sof\sOOM\shandling\sin\sthe\sjson\sextension. +D 2015-08-22T03:05:54.585 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c f83f02ec4d7cc18794bd7ac046f1bb5905805b44 +F ext/misc/json1.c 2eae688e07700b94ed7e03051a90918c6744d0a9 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 178f9a352c6c9e15e809e1a47530c6592d18578d -R 3f8fa8b32979f76f9659e6c12a12d079 +P 351bc22fa9b5a2e50da3583a882c5aa390bda19f +R ca8b5b15c77432f02507718e8d7e076f U drh -Z fe48d12321db6aa71a1dc8fdceeea2c3 +Z d373dd42adbb610fe45611bf2ca7212e diff --git a/manifest.uuid b/manifest.uuid index 661f57a8ab..4064ce58f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -351bc22fa9b5a2e50da3583a882c5aa390bda19f \ No newline at end of file +213a6c5ccbcfe4495d45e3608e99a6398751aeed \ No newline at end of file From bc8f092ca10493d59ba619fb5a9fe6a7358e3372 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Aug 2015 19:39:04 +0000 Subject: [PATCH 26/54] Add the json_valid() function to the json1.c extension. Fix various minor problems in the json1.c extension. FossilOrigin-Name: 380a97345b446214843a63ccc017d49a52d884da --- ext/misc/json1.c | 188 +++++++++++++++++++++++++++++++---------------- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 132 insertions(+), 70 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index ea456bfeac..a68ef7469c 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -573,18 +573,19 @@ static int jsonParseAddNode( static int jsonParseValue(JsonParse *pParse, u32 i){ char c; u32 j; - u32 iThis; + int iThis; int x; while( isspace(pParse->zJson[i]) ){ i++; } if( (c = pParse->zJson[i])==0 ) return 0; if( c=='{' ){ /* Parse object */ iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + if( iThis<0 ) return -1; for(j=i+1;;j++){ while( isspace(pParse->zJson[j]) ){ j++; } x = jsonParseValue(pParse, j); if( x<0 ){ - if( x==(-2) && pParse->nNode==iThis+1 ) return j+1; + if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; return -1; } if( pParse->oom ) return -1; @@ -602,16 +603,17 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c!='}' ) return -1; break; } - pParse->aNode[iThis].n = pParse->nNode - iThis - 1; + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; return j+1; }else if( c=='[' ){ /* Parse array */ iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + if( iThis<0 ) return -1; for(j=i+1;;j++){ while( isspace(pParse->zJson[j]) ){ j++; } x = jsonParseValue(pParse, j); if( x<0 ){ - if( x==(-3) && pParse->nNode==iThis+1 ) return j+1; + if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; return -1; } j = x; @@ -621,7 +623,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c!=']' ) return -1; break; } - pParse->aNode[iThis].n = pParse->nNode - iThis - 1; + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; return j+1; }else if( c=='"' ){ /* Parse string */ @@ -701,7 +703,11 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ ** ** pParse is uninitialized when this routine is called. */ -static int jsonParse(JsonParse *pParse, const char *zJson){ +static int jsonParse( + JsonParse *pParse, /* Initialize and fill this JsonParse object */ + sqlite3_context *pCtx, /* Report errors here */ + const char *zJson /* Input JSON text to be parsed */ +){ int i; if( zJson==0 ) return 1; memset(pParse, 0, sizeof(*pParse)); @@ -712,6 +718,7 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ if( zJson[i] ) i = -1; } if( i<0 ){ + if( pParse->oom && pCtx!=0 ) sqlite3_result_error_nomem(pCtx); jsonParseReset(pParse); return 1; } @@ -775,7 +782,7 @@ static JsonNode *jsonLookup( const char *zPath, /* The path to search */ int *pApnd /* Append nodes to complete path if not NULL */ ){ - u32 i, j, k, nKey; + u32 i, j, nKey; const char *zKey; JsonNode *pRoot = &pParse->aNode[iRoot]; if( zPath[0]==0 ) return pRoot; @@ -810,13 +817,20 @@ static JsonNode *jsonLookup( j = 1; } if( pApnd ){ - k = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - pRoot->u.iAppend = k - iRoot; - pRoot->jnFlags |= JNODE_APPEND; - k = jsonParseAddNode(pParse, JSON_STRING, i, zPath); - if( !pParse->oom ) pParse->aNode[k].jnFlags |= JNODE_RAW; + u32 iStart, iLabel; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + iLabel = jsonParseAddNode(pParse, JSON_STRING, i, zPath); zPath += i; - return jsonLookupAppend(pParse, zPath, pApnd); + pNode = jsonLookupAppend(pParse, zPath, pApnd); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + pParse->aNode[iLabel].jnFlags |= JNODE_RAW; + } + return pNode; } }else if( zPath[0]=='[' && isdigit(zPath[1]) ){ if( pRoot->eType!=JSON_ARRAY ) return 0; @@ -830,9 +844,9 @@ static JsonNode *jsonLookup( zPath++; j = 1; for(;;){ - while( i>0 && j<=pRoot->n ){ + while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; j += jsonNodeSize(&pRoot[j]); - i--; } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; iRoot += pRoot->u.iAppend; @@ -843,17 +857,25 @@ static JsonNode *jsonLookup( return jsonLookup(pParse, iRoot+j, zPath, pApnd); } if( i==0 && pApnd ){ - k = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); - pRoot->u.iAppend = k - iRoot; - pRoot->jnFlags |= JNODE_APPEND; - return jsonLookupAppend(pParse, zPath, pApnd); + u32 iStart; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); + pNode = jsonLookupAppend(pParse, zPath, pApnd); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + } + return pNode; } } return 0; } /* -** Append content to pParse that will complete zPath. +** Append content to pParse that will complete zPath. Return a pointer +** to the inserted node, or return NULL if the append fails. */ static JsonNode *jsonLookupAppend( JsonParse *pParse, /* Append content to the JSON parse */ @@ -876,6 +898,19 @@ static JsonNode *jsonLookupAppend( return jsonLookup(pParse, pParse->nNode-1, zPath, pApnd); } +/* +** Report the wrong number of arguments for json_insert(), json_replace() +** or json_set(). +*/ +static void jsonWrongNumArgs( + sqlite3_context *pCtx, + const char *zFuncName +){ + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", + zFuncName); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); +} /**************************************************************************** ** SQL functions used for testing and debugging @@ -888,7 +923,7 @@ static JsonNode *jsonLookupAppend( ** well-formed. */ static void jsonParseFunc( - sqlite3_context *context, + sqlite3_context *ctx, int argc, sqlite3_value **argv ){ @@ -898,8 +933,8 @@ static void jsonParseFunc( char zBuf[100]; assert( argc==1 ); - if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonInit(&s, context); + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonInit(&s, ctx); for(i=0; ijnFlags |= JNODE_REMOVE; } if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturn(x.aNode, context, 0); + jsonReturn(x.aNode, ctx, 0); } } jsonParseReset(&x); @@ -1121,7 +1156,7 @@ static void jsonRemoveFunc( ** this routine is a no-op. If JSON is ill-formed, return NULL. */ static void jsonReplaceFunc( - sqlite3_context *context, + sqlite3_context *ctx, int argc, sqlite3_value **argv ){ @@ -1132,11 +1167,10 @@ static void jsonReplaceFunc( if( argc<1 ) return; if( (argc&1)==0 ) { - sqlite3_result_error(context, - "json_replace() needs an odd number of arguments", -1); + jsonWrongNumArgs(ctx, "replace"); return; } - if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); @@ -1149,9 +1183,9 @@ static void jsonReplaceFunc( } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(context, argv[x.aNode[0].iVal]); + sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); }else{ - jsonReturn(x.aNode, context, argv); + jsonReturn(x.aNode, ctx, argv); } } jsonParseReset(&x); @@ -1170,7 +1204,7 @@ static void jsonReplaceFunc( ** routine is a no-op. If JSON is ill-formed, return NULL. */ static void jsonSetFunc( - sqlite3_context *context, + sqlite3_context *ctx, int argc, sqlite3_value **argv ){ @@ -1179,15 +1213,14 @@ static void jsonSetFunc( const char *zPath; u32 i; int bApnd; - int bIsSet = *(int*)sqlite3_user_data(context); + int bIsSet = *(int*)sqlite3_user_data(ctx); if( argc<1 ) return; if( (argc&1)==0 ) { - sqlite3_result_error(context, - "json_set() needs an odd number of arguments", -1); + jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } - if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); @@ -1195,17 +1228,21 @@ static void jsonSetFunc( if( zPath[0]!='$' ) continue; bApnd = 0; pNode = jsonLookup(&x, 0, &zPath[1], &bApnd); - if( pNode && (bApnd || bIsSet) ){ + if( x.oom ){ + sqlite3_result_error_nomem(ctx); + goto jsonSetDone; + }else if( pNode && (bApnd || bIsSet) ){ pNode->jnFlags |= JNODE_REPLACE; pNode->iVal = i+1; } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(context, argv[x.aNode[0].iVal]); + sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); }else{ - jsonReturn(x.aNode, context, argv); + jsonReturn(x.aNode, ctx, argv); } } +jsonSetDone: jsonParseReset(&x); } @@ -1217,7 +1254,7 @@ static void jsonSetFunc( ** input is not a well-formed JSON string. */ static void jsonTypeFunc( - sqlite3_context *context, + sqlite3_context *ctx, int argc, sqlite3_value **argv ){ @@ -1232,15 +1269,39 @@ static void jsonTypeFunc( }else{ zPath = 0; } - if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ JsonNode *pNode = x.aNode; if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); - sqlite3_result_text(context, jsonType[pNode->eType], -1, SQLITE_STATIC); + if( pNode ){ + sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); + } } jsonParseReset(&x); } +/* +** json_valid(JSON) +** +** Return 1 if JSON is a valid JSON string. Return 0 otherwise. +*/ +static void jsonValidFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + int rc = 0; + + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]))==0 + && x.nNode>0 + ){ + rc = 1; + } + jsonParseReset(&x); + sqlite3_result_int(ctx, rc); +} + /**************************************************************************** ** The json_each virtual table ****************************************************************************/ @@ -1588,7 +1649,7 @@ static int jsonEachFilter( p->zJson = sqlite3_malloc64( n+1 ); if( p->zJson==0 ) return SQLITE_NOMEM; memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, p->zJson) + if( jsonParse(&p->sParse, 0, p->zJson) || (p->bRecursive && jsonParseFindParents(&p->sParse)) ){ jsonEachCursorReset(p); @@ -1617,7 +1678,7 @@ static int jsonEachFilter( p->iEnd = p->i+1; } } - return SQLITE_OK; + return p->sParse.oom ? SQLITE_NOMEM : SQLITE_OK; } /* The methods of the json_each virtual table */ @@ -1707,6 +1768,7 @@ int sqlite3_json_init( { "json_set", -1, 1, jsonSetFunc }, { "json_type", 1, 0, jsonTypeFunc }, { "json_type", 2, 0, jsonTypeFunc }, + { "json_valid", 1, 0, jsonValidFunc }, #if SQLITE_DEBUG /* DEBUG and TESTING functions */ diff --git a/manifest b/manifest index de7c994cbe..f08dd2f05a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sinstances\sof\sOOM\shandling\sin\sthe\sjson\sextension. -D 2015-08-22T03:05:54.585 +C Add\sthe\sjson_valid()\sfunction\sto\sthe\sjson1.c\sextension.\s\sFix\svarious\sminor\nproblems\sin\sthe\sjson1.c\sextension. +D 2015-08-22T19:39:04.060 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 2eae688e07700b94ed7e03051a90918c6744d0a9 +F ext/misc/json1.c 88273dda16ac505b898be5ba90b29aaabbe37e71 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 351bc22fa9b5a2e50da3583a882c5aa390bda19f -R ca8b5b15c77432f02507718e8d7e076f +P 213a6c5ccbcfe4495d45e3608e99a6398751aeed +R 71acf7e81664c832e0217a769189f3b7 U drh -Z d373dd42adbb610fe45611bf2ca7212e +Z 942f59d21b0c9aa639cfef239bb85f7f diff --git a/manifest.uuid b/manifest.uuid index 4064ce58f6..db562f929c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -213a6c5ccbcfe4495d45e3608e99a6398751aeed \ No newline at end of file +380a97345b446214843a63ccc017d49a52d884da \ No newline at end of file From 8784eca17f029b709852e8ca8aabfe73d7baf6f1 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 23 Aug 2015 02:42:30 +0000 Subject: [PATCH 27/54] Fixes to json_each() and json_tree(). Improved json_parse() debugging output. FossilOrigin-Name: fc1b24f316af07a64672f6edc14ebcff487dffbb --- ext/misc/json1.c | 54 ++++++++++++++++++++++++------------------------ manifest | 12 +++++------ manifest.uuid | 2 +- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index a68ef7469c..d5d7d5a96c 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -195,14 +195,6 @@ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ p->nUsed += (int)strlen(p->zBuf+p->nUsed); } -#ifdef SQLITE_DEBUG -/* Append the zero-terminated string zIn -*/ -static void jsonAppend(JsonString *p, const char *zIn){ - jsonAppendRaw(p, zIn, (u32)strlen(zIn)); -} -#endif - /* Append a single character */ static void jsonAppendChar(JsonString *p, char c){ @@ -468,14 +460,12 @@ static void jsonReturn( c = z[++i]; if( c=='u' && z[1] ){ u32 v = 0, k; - z++; - for(k=0; k<4 && z[k]; k++){ - c = z[0]; + for(k=0; k<4 && z[i+1]; i++, k++){ + c = z[i+1]; if( c>='0' && c<='9' ) v = v*16 + c - '0'; else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10; else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10; else break; - z++; } if( v<=0x7f ){ zOut[j++] = v; @@ -678,7 +668,11 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( seenE ) return -1; seenDP = seenE = 1; c = pParse->zJson[j+1]; - if( c=='+' || c=='-' ) j++; + if( c=='+' || c=='-' ){ + j++; + c = pParse->zJson[j+1]; + } + if( c<'0' || c>'0' ) return -1; continue; } break; @@ -837,7 +831,7 @@ static JsonNode *jsonLookup( i = 0; zPath++; while( isdigit(zPath[0]) ){ - i = i + zPath[0] - '0'; + i = i*10 + zPath[0] - '0'; zPath++; } if( zPath[0]!=']' ) return 0; @@ -930,15 +924,14 @@ static void jsonParseFunc( JsonString s; /* Output string - not real JSON */ JsonParse x; /* The parse */ u32 i; - char zBuf[100]; assert( argc==1 ); if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonParseFindParents(&x); jsonInit(&s, ctx); for(i=0; ibRecursive ){ if( p->i==0 ){ p->i = 1; - }else if( p->sParse.aNode[p->sParse.aUp[p->i]].eType==JSON_OBJECT ){ - p->i += 2; }else{ + u32 iUp = p->sParse.aUp[p->i]; + JsonNode *pUp = &p->sParse.aNode[iUp]; p->i++; + if( pUp->eType==JSON_OBJECT && (pUp->n + iUp >= p->i) ) p->i++; } p->iRowid++; if( p->isParse.nNode ){ - JsonNode *pUp = &p->sParse.aNode[p->sParse.aUp[p->i]]; + u32 iUp = p->sParse.aUp[p->i]; + JsonNode *pUp = &p->sParse.aNode[iUp]; p->eType = pUp->eType; - if( pUp->eType==JSON_ARRAY ) pUp->u.iKey++; - if( p->sParse.aNode[p->i].eType==JSON_ARRAY ){ - p->sParse.aNode[p->i].u.iKey = 0; + if( pUp->eType==JSON_ARRAY ){ + if( iUp==p->i-1 ){ + pUp->u.iKey = 0; + }else{ + pUp->u.iKey++; + } } } }else{ @@ -1490,13 +1488,14 @@ static int jsonEachColumn( JsonNode *pThis = &p->sParse.aNode[p->i]; switch( i ){ case JEACH_KEY: { + if( p->i==0 ) break; if( p->eType==JSON_OBJECT ){ jsonReturn(pThis, ctx, 0); }else if( p->eType==JSON_ARRAY ){ u32 iKey; if( p->bRecursive ){ if( p->iRowid==0 ) break; - iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey - 1; + iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; }else{ iKey = p->iRowid; } @@ -1505,7 +1504,7 @@ static int jsonEachColumn( break; } case JEACH_VALUE: { - if( p->eType==JSON_OBJECT ) pThis++; + if( p->eType==JSON_OBJECT && p->i>0 ) pThis++; jsonReturn(pThis, ctx, 0); break; } @@ -1672,7 +1671,8 @@ static int jsonEachFilter( p->i = (int)(pNode - p->sParse.aNode); p->eType = pNode->eType; if( p->eType>=JSON_ARRAY ){ - p->i++; + pNode->u.iKey = 0; + if( !p->bRecursive ) p->i++; p->iEnd = p->i + pNode->n; }else{ p->iEnd = p->i+1; diff --git a/manifest b/manifest index f08dd2f05a..34b5e70a87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sjson_valid()\sfunction\sto\sthe\sjson1.c\sextension.\s\sFix\svarious\sminor\nproblems\sin\sthe\sjson1.c\sextension. -D 2015-08-22T19:39:04.060 +C Fixes\sto\sjson_each()\sand\sjson_tree().\s\sImproved\sjson_parse()\sdebugging\soutput. +D 2015-08-23T02:42:30.942 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 88273dda16ac505b898be5ba90b29aaabbe37e71 +F ext/misc/json1.c 31bc1babd31190203cb86fcdbe21522756f65b12 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 213a6c5ccbcfe4495d45e3608e99a6398751aeed -R 71acf7e81664c832e0217a769189f3b7 +P 380a97345b446214843a63ccc017d49a52d884da +R a54dc4e155baa0caa5702a3578af52f1 U drh -Z 942f59d21b0c9aa639cfef239bb85f7f +Z b71b5a39d2829e2cd3b83f4646e372f9 diff --git a/manifest.uuid b/manifest.uuid index db562f929c..29702aa763 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -380a97345b446214843a63ccc017d49a52d884da \ No newline at end of file +fc1b24f316af07a64672f6edc14ebcff487dffbb \ No newline at end of file From c3722b2103f104a7d511c9fe4a831fb81c15c06c Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 23 Aug 2015 20:44:59 +0000 Subject: [PATCH 28/54] Fix minor glitches in the json1.c extension, mostly having to do with OOM behavior. FossilOrigin-Name: cc5204149c4053b9e529a72102d8df0925ad1ea1 --- ext/misc/json1.c | 10 +++++++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index d5d7d5a96c..ca22928302 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -703,10 +703,11 @@ static int jsonParse( const char *zJson /* Input JSON text to be parsed */ ){ int i; - if( zJson==0 ) return 1; memset(pParse, 0, sizeof(*pParse)); + if( zJson==0 ) return 1; pParse->zJson = zJson; i = jsonParseValue(pParse, 0); + if( pParse->oom ) i = -1; if( i>0 ){ while( isspace(zJson[i]) ) i++; if( zJson[i] ) i = -1; @@ -753,7 +754,10 @@ static int jsonParseFindParents(JsonParse *pParse){ u32 *aUp; assert( pParse->aUp==0 ); aUp = pParse->aUp = sqlite3_malloc( sizeof(u32)*pParse->nNode ); - if( aUp==0 ) return SQLITE_NOMEM; + if( aUp==0 ){ + pParse->oom = 1; + return SQLITE_NOMEM; + } jsonParseFillInParentage(pParse, 0, 0); return SQLITE_OK; } @@ -1672,8 +1676,8 @@ static int jsonEachFilter( p->eType = pNode->eType; if( p->eType>=JSON_ARRAY ){ pNode->u.iKey = 0; + p->iEnd = p->i + pNode->n + 1; if( !p->bRecursive ) p->i++; - p->iEnd = p->i + pNode->n; }else{ p->iEnd = p->i+1; } diff --git a/manifest b/manifest index 34b5e70a87..7429fea7cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\sjson_each()\sand\sjson_tree().\s\sImproved\sjson_parse()\sdebugging\soutput. -D 2015-08-23T02:42:30.942 +C Fix\sminor\sglitches\sin\sthe\sjson1.c\sextension,\smostly\shaving\sto\sdo\swith\sOOM\nbehavior. +D 2015-08-23T20:44:59.358 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 31bc1babd31190203cb86fcdbe21522756f65b12 +F ext/misc/json1.c 4b66d2ce1dc458e214d0f3663a3870e592484230 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 380a97345b446214843a63ccc017d49a52d884da -R a54dc4e155baa0caa5702a3578af52f1 +P fc1b24f316af07a64672f6edc14ebcff487dffbb +R f3f5a4cd7433351edd3f5e216e8c6ba5 U drh -Z b71b5a39d2829e2cd3b83f4646e372f9 +Z 19ea95791c67f4fad1659a43be075b46 diff --git a/manifest.uuid b/manifest.uuid index 29702aa763..db5d937671 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc1b24f316af07a64672f6edc14ebcff487dffbb \ No newline at end of file +cc5204149c4053b9e529a72102d8df0925ad1ea1 \ No newline at end of file From e792b5b4206a50fcd3ba96cc653329e9f27dc9bc Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 23 Aug 2015 20:48:29 +0000 Subject: [PATCH 29/54] Fix a comment typo on sqlite3ExprAlloc(). No code changes. FossilOrigin-Name: 518d6220a12fb1289f699ef0821e6adfcd286ed0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7429fea7cd..b3470ed66a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\sglitches\sin\sthe\sjson1.c\sextension,\smostly\shaving\sto\sdo\swith\sOOM\nbehavior. -D 2015-08-23T20:44:59.358 +C Fix\sa\scomment\stypo\son\ssqlite3ExprAlloc().\s\sNo\scode\schanges. +D 2015-08-23T20:48:29.272 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c 8ec787fed4929d8ccdf6b1bc360fccc3e1d2ca58 F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a F src/delete.c 8857a6f27560718f65d43bdbec86c967ae1f8dfa -F src/expr.c 9b9fa7f825290dee945007edc9fe8fdd9b8ce49e +F src/expr.c c34408e0ac1c57cf58ffa2003b339715ee9bda57 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c c9b63a217d86582c22121699a47f22f524608869 F src/func.c 824bea430d3a2b7dbc62806ad54da8fdb8ed9e3f @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fc1b24f316af07a64672f6edc14ebcff487dffbb -R f3f5a4cd7433351edd3f5e216e8c6ba5 +P cc5204149c4053b9e529a72102d8df0925ad1ea1 +R 45301f1702fbfadcb1473edb5c388742 U drh -Z 19ea95791c67f4fad1659a43be075b46 +Z a9752167ae3428587580d2325e13d760 diff --git a/manifest.uuid b/manifest.uuid index db5d937671..78f168c777 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc5204149c4053b9e529a72102d8df0925ad1ea1 \ No newline at end of file +518d6220a12fb1289f699ef0821e6adfcd286ed0 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1062733cb9..04cd36ea79 100644 --- a/src/expr.c +++ b/src/expr.c @@ -433,7 +433,7 @@ void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** is responsible for making sure the node eventually gets freed. ** ** If dequote is true, then the token (if it exists) is dequoted. -** If dequote is false, no dequoting is performance. The deQuote +** If dequote is false, no dequoting is performed. The deQuote ** parameter is ignored if pToken is NULL or if the token does not ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. From 442a7c601505dc0aec141b1b05450ddb530562e4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Aug 2015 02:32:04 +0000 Subject: [PATCH 30/54] Fix corner-case problems in the type and atom columns of json_each() and json_tree(). FossilOrigin-Name: f0aba0e120074430cd7ad93291fcc97b8a25a054 --- ext/misc/json1.c | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index ca22928302..4945826b93 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -1513,12 +1513,12 @@ static int jsonEachColumn( break; } case JEACH_TYPE: { - if( p->eType==JSON_OBJECT ) pThis++; + if( p->eType==JSON_OBJECT && p->i>0 ) pThis++; sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); break; } case JEACH_ATOM: { - if( p->eType==JSON_OBJECT ) pThis++; + if( p->eType==JSON_OBJECT && p->i>0 ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; jsonReturn(pThis, ctx, 0); break; diff --git a/manifest b/manifest index b3470ed66a..86f1a81da0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scomment\stypo\son\ssqlite3ExprAlloc().\s\sNo\scode\schanges. -D 2015-08-23T20:48:29.272 +C Fix\scorner-case\sproblems\sin\sthe\stype\sand\satom\scolumns\sof\sjson_each()\sand\njson_tree(). +D 2015-08-24T02:32:04.608 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 4b66d2ce1dc458e214d0f3663a3870e592484230 +F ext/misc/json1.c 443f8b54b76042112f669c694683f7ecb2512580 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cc5204149c4053b9e529a72102d8df0925ad1ea1 -R 45301f1702fbfadcb1473edb5c388742 +P 518d6220a12fb1289f699ef0821e6adfcd286ed0 +R d6c92e3d9acd1a46fd49a9ba005fd842 U drh -Z a9752167ae3428587580d2325e13d760 +Z 8270e043900b276b537309af17df590f diff --git a/manifest.uuid b/manifest.uuid index 78f168c777..f0c18aac37 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -518d6220a12fb1289f699ef0821e6adfcd286ed0 \ No newline at end of file +f0aba0e120074430cd7ad93291fcc97b8a25a054 \ No newline at end of file From 80d874083b38c5e24d854a76ab07b791fb0dc2cb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Aug 2015 12:42:41 +0000 Subject: [PATCH 31/54] Improvements to JSON string dequoting. FossilOrigin-Name: 196d66d34d9783622e6f2f79eafea1488fc6f5cf --- ext/misc/json1.c | 14 +++++--------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 4945826b93..cd4531bccb 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -454,33 +454,29 @@ static void jsonReturn( } for(i=1, j=0; i='0' && c<='9' ) v = v*16 + c - '0'; else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10; else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10; else break; } + if( v==0 ) break; if( v<=0x7f ){ zOut[j++] = v; }else if( v<=0x7ff ){ zOut[j++] = 0xc0 | (v>>6); zOut[j++] = 0x80 | (v&0x3f); - }else if( v<=0xffff ){ + }else{ zOut[j++] = 0xe0 | (v>>12); zOut[j++] = 0x80 | ((v>>6)&0x3f); zOut[j++] = 0x80 | (v&0x3f); - }else if( v<=0x10ffff ){ - zOut[j++] = 0xf0 | (v>>18); - zOut[j++] = 0x80 | ((v>>12)&0x3f); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); } }else{ if( c=='b' ){ diff --git a/manifest b/manifest index 86f1a81da0..d46e10f6c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scorner-case\sproblems\sin\sthe\stype\sand\satom\scolumns\sof\sjson_each()\sand\njson_tree(). -D 2015-08-24T02:32:04.608 +C Improvements\sto\sJSON\sstring\sdequoting. +D 2015-08-24T12:42:41.080 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 443f8b54b76042112f669c694683f7ecb2512580 +F ext/misc/json1.c 541004e47235cefc2843ab03c100517452931913 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 518d6220a12fb1289f699ef0821e6adfcd286ed0 -R d6c92e3d9acd1a46fd49a9ba005fd842 +P f0aba0e120074430cd7ad93291fcc97b8a25a054 +R 3625d17566ea8cc316dd4952d6bffb99 U drh -Z 8270e043900b276b537309af17df590f +Z 6090b9f6d755a9917a579931aff72e9d diff --git a/manifest.uuid b/manifest.uuid index f0c18aac37..bf64eaf064 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f0aba0e120074430cd7ad93291fcc97b8a25a054 \ No newline at end of file +196d66d34d9783622e6f2f79eafea1488fc6f5cf \ No newline at end of file From bc622bc045a919b491be2d1ba0c56fd5fd7ff22f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Aug 2015 15:39:42 +0000 Subject: [PATCH 32/54] Disallow the use of COLLATE clauses and the ASC and DESC keywords within foreign key constraints and in the argument list to common table expressions. FossilOrigin-Name: 83cbc4d8761498647794affffa961a4fca311be7 --- manifest | 19 ++++++++-------- manifest.uuid | 2 +- src/build.c | 35 ++++++++++++++++++++++++++-- src/expr.c | 15 ++++++++++++ src/parse.y | 10 ++++---- src/sqliteInt.h | 4 ++++ test/parser1.test | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 126 insertions(+), 17 deletions(-) create mode 100644 test/parser1.test diff --git a/manifest b/manifest index d46e10f6c8..9b3d236121 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sJSON\sstring\sdequoting. -D 2015-08-24T12:42:41.080 +C Disallow\sthe\suse\sof\sCOLLATE\sclauses\sand\sthe\sASC\sand\sDESC\skeywords\swithin\nforeign\skey\sconstraints\sand\sin\sthe\sargument\slist\sto\scommon\stable\sexpressions. +D 2015-08-24T15:39:42.405 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,14 +282,14 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c 5eb5d055a1d1cdaaea25e01b12607aa894bc0911 +F src/build.c f49c55c1fba430c2d6af5039d0bf14de5ae6a427 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c 8ec787fed4929d8ccdf6b1bc360fccc3e1d2ca58 F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a F src/delete.c 8857a6f27560718f65d43bdbec86c967ae1f8dfa -F src/expr.c c34408e0ac1c57cf58ffa2003b339715ee9bda57 +F src/expr.c c05d67f1a03c097d5c29839d5a538cfde9c472ce F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c c9b63a217d86582c22121699a47f22f524608869 F src/func.c 824bea430d3a2b7dbc62806ad54da8fdb8ed9e3f @@ -326,7 +326,7 @@ F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c aa916ca28606ccf4b6877dfc2b643ccbca86589f F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 -F src/parse.y ad9af8552f6f340bd646577ca63356a6f82b6a7e +F src/parse.y b5e0a5f8bb9ec33a58aa34850b6fae46aac51fdd F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c a3fe31b17e841ec70beee72a2c960e9c787a8857 @@ -342,7 +342,7 @@ F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h 89e68539d645db597366a91411468b51e73c21a0 +F src/sqliteInt.h 1741691824491c330dda1e49f370920bfd3aa230 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -910,6 +910,7 @@ F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 +F test/parser1.test 23867b6f2c4758c7774108826d9f17e9cd17bcde F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff @@ -1378,7 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f0aba0e120074430cd7ad93291fcc97b8a25a054 -R 3625d17566ea8cc316dd4952d6bffb99 +P 196d66d34d9783622e6f2f79eafea1488fc6f5cf +R 2a74528f51d8172b5387a3b7f110bb2b U drh -Z 6090b9f6d755a9917a579931aff72e9d +Z ffc4f8e604c67b08cae8ed1eda808a43 diff --git a/manifest.uuid b/manifest.uuid index bf64eaf064..9ca8f1fa75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -196d66d34d9783622e6f2f79eafea1488fc6f5cf \ No newline at end of file +83cbc4d8761498647794affffa961a4fca311be7 \ No newline at end of file diff --git a/src/build.c b/src/build.c index e45908dc3b..0a816b398e 100644 --- a/src/build.c +++ b/src/build.c @@ -1321,7 +1321,7 @@ void sqlite3AddPrimaryKey( } if( nTerm==1 && zType && sqlite3StrICmp(zType, "INTEGER")==0 - && sortOrder==SQLITE_SO_ASC + && sortOrder!=SQLITE_SO_DESC ){ pTab->iPKey = iCol; pTab->keyConf = (u8)onError; @@ -2600,6 +2600,8 @@ void sqlite3CreateForeignKey( assert( pTo!=0 ); if( p==0 || IN_DECLARE_VTAB ) goto fk_end; + sqlite3RestrictColumnListSyntax(pParse, pFromCol); + sqlite3RestrictColumnListSyntax(pParse, pToCol); if( pFromCol==0 ){ int iCol = p->nCol-1; if( NEVER(iCol<0) ) goto fk_end; @@ -3038,7 +3040,8 @@ Index *sqlite3CreateIndex( if( pList==0 ) goto exit_create_index; pList->a[0].zName = sqlite3DbStrDup(pParse->db, pTab->aCol[pTab->nCol-1].zName); - pList->a[0].sortOrder = (u8)sortOrder; + assert( pList->nExpr==1 ); + sqlite3ExprListSetSortOrder(pList, sortOrder); } /* Figure out how many bytes of space are required to store explicitly @@ -4283,6 +4286,32 @@ KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ return pKey; } +/* +** Generate a syntax error if the expression list provided contains +** any COLLATE or ASC or DESC keywords. +** +** Some legacy versions of SQLite allowed constructs like: +** +** CREATE TABLE x(..., FOREIGN KEY(x COLLATE binary DESC) REFERENCES...); +** ^^^^^^^^^^^^^^^^^^^ +** +** The COLLATE and sort order terms were ignored. To prevent compatibility +** problems in case something like this appears in a legacy sqlite_master +** table, only enforce the restriction on new SQL statements, not when +** parsing the schema out of the sqlite_master table. +*/ +void sqlite3RestrictColumnListSyntax(Parse *pParse, ExprList *p){ + int i; + if( p==0 || pParse->db->init.busy ) return; + for(i=0; inExpr; i++){ + if( p->a[i].pExpr!=0 || p->a[i].bDefinedSO ){ + sqlite3ErrorMsg(pParse, "syntax error after column name \"%w\"", + p->a[i].zName); + return; + } + } +} + #ifndef SQLITE_OMIT_CTE /* ** This routine is invoked once per CTE by the parser while parsing a @@ -4299,6 +4328,8 @@ With *sqlite3WithAdd( With *pNew; char *zName; + sqlite3RestrictColumnListSyntax(pParse, pArglist); + /* Check that the CTE name is unique within this WITH clause. If ** not, store an error in the Parse structure. */ zName = sqlite3NameFromToken(pParse->db, pName); diff --git a/src/expr.c b/src/expr.c index 04cd36ea79..1c57ecc6fd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1160,6 +1160,21 @@ no_mem: return 0; } +/* +** Set the sort order for the last element on the given ExprList. +*/ +void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ + if( p==0 ) return; + assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC>=0 && SQLITE_SO_DESC>0 ); + assert( p->nExpr>0 ); + if( iSortOrder<0 ){ + assert( p->a[p->nExpr-1].sortOrder==SQLITE_SO_ASC ); + return; + } + p->a[p->nExpr-1].sortOrder = (u8)iSortOrder; + p->a[p->nExpr-1].bDefinedSO = 1; +} + /* ** Set the ExprList.a[].zName element of the most recently added item ** on the expression list. diff --git a/src/parse.y b/src/parse.y index 3d186b28aa..bc193ec1f2 100644 --- a/src/parse.y +++ b/src/parse.y @@ -680,18 +680,18 @@ orderby_opt(A) ::= . {A = 0;} orderby_opt(A) ::= ORDER BY sortlist(X). {A = X;} sortlist(A) ::= sortlist(X) COMMA expr(Y) sortorder(Z). { A = sqlite3ExprListAppend(pParse,X,Y.pExpr); - if( A ) A->a[A->nExpr-1].sortOrder = (u8)Z; + sqlite3ExprListSetSortOrder(A,Z); } sortlist(A) ::= expr(Y) sortorder(Z). { A = sqlite3ExprListAppend(pParse,0,Y.pExpr); - if( A && ALWAYS(A->a) ) A->a[0].sortOrder = (u8)Z; + sqlite3ExprListSetSortOrder(A,Z); } %type sortorder {int} sortorder(A) ::= ASC. {A = SQLITE_SO_ASC;} sortorder(A) ::= DESC. {A = SQLITE_SO_DESC;} -sortorder(A) ::= . {A = SQLITE_SO_ASC;} +sortorder(A) ::= . {A = SQLITE_SO_UNDEFINED;} %type groupby_opt {ExprList*} %destructor groupby_opt {sqlite3ExprListDelete(pParse->db, $$);} @@ -1229,14 +1229,14 @@ idxlist(A) ::= idxlist(X) COMMA nm(Y) collate(C) sortorder(Z). { A = sqlite3ExprListAppend(pParse,X, p); sqlite3ExprListSetName(pParse,A,&Y,1); sqlite3ExprListCheckLength(pParse, A, "index"); - if( A ) A->a[A->nExpr-1].sortOrder = (u8)Z; + sqlite3ExprListSetSortOrder(A,Z); } idxlist(A) ::= nm(Y) collate(C) sortorder(Z). { Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C, 1); A = sqlite3ExprListAppend(pParse,0, p); sqlite3ExprListSetName(pParse, A, &Y, 1); sqlite3ExprListCheckLength(pParse, A, "index"); - if( A ) A->a[A->nExpr-1].sortOrder = (u8)Z; + sqlite3ExprListSetSortOrder(A,Z); } %type collate {Token} diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 96a7700210..18a0fd705f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1525,6 +1525,7 @@ struct CollSeq { */ #define SQLITE_SO_ASC 0 /* Sort in ascending order */ #define SQLITE_SO_DESC 1 /* Sort in ascending order */ +#define SQLITE_SO_UNDEFINED -1 /* No sort order specified */ /* ** Column affinity types. @@ -2189,6 +2190,7 @@ struct ExprList { unsigned done :1; /* A flag to indicate when processing is finished */ unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ unsigned reusable :1; /* Constant expression is reusable */ + unsigned bDefinedSO :1; /* True if either DESC or ASC keywords present */ union { struct { u16 iOrderByCol; /* For ORDER BY, column number in result set */ @@ -3244,6 +3246,7 @@ Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); void sqlite3ExprAssignVarNumber(Parse*, Expr*); void sqlite3ExprDelete(sqlite3*, Expr*); ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); +void sqlite3ExprListSetSortOrder(ExprList*,int); void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); void sqlite3ExprListDelete(sqlite3*, ExprList*); @@ -3755,6 +3758,7 @@ const char *sqlite3JournalModename(int); int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #endif +void sqlite3RestrictColumnListSyntax(Parse*,ExprList*); #ifndef SQLITE_OMIT_CTE With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*); void sqlite3WithDelete(sqlite3*,With*); diff --git a/test/parser1.test b/test/parser1.test new file mode 100644 index 0000000000..f4c3227f87 --- /dev/null +++ b/test/parser1.test @@ -0,0 +1,58 @@ +# 2014-08-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# The focus of this script is testing details of the SQL language parser. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_catchsql_test parser1-1.1 { + CREATE TABLE t1( + a TEXT PRIMARY KEY, + b TEXT, + FOREIGN KEY(b COLLATE nocase DESC) REFERENCES t1(a COLLATE binary ASC) + ); +} {1 {syntax error after column name "a"}} +do_execsql_test parser1-1.2 { + CREATE TABLE t1( + a TEXT PRIMARY KEY, + b TEXT, + FOREIGN KEY(b) REFERENCES t1(a) + ); + INSERT INTO t1 VALUES('abc',NULL),('xyz','abc'); + PRAGMA writable_schema=on; + UPDATE sqlite_master SET sql='CREATE TABLE t1( + a TEXT PRIMARY KEY, + b TEXT, + FOREIGN KEY(b COLLATE nocase) REFERENCES t1(a) + )' WHERE name='t1'; + SELECT name FROM sqlite_master WHERE sql LIKE '%collate%'; +} {t1} +sqlite3 db2 test.db +do_test parser1-1.3 { + sqlite3 db2 test.db + db2 eval {SELECT * FROM t1 ORDER BY 1} +} {abc {} xyz abc} + + +do_catchsql_test parser1-2.1 { + WITH RECURSIVE + c(x COLLATE binary) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<5) + SELECT x FROM c; +} {1 {syntax error after column name "x"}} +do_catchsql_test parser1-2.2 { + WITH RECURSIVE + c(x ASC) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<5) + SELECT x FROM c; +} {1 {syntax error after column name "x"}} + +finish_test From a09767b230e858f5fdb331532d3a73ef5f7137c2 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 24 Aug 2015 17:18:43 +0000 Subject: [PATCH 33/54] Enhancements to the batch build tool for MSVC. FossilOrigin-Name: a1ae20cd97456a1126cfa1a9bedce0bac0940ad6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/build-all-msvc.bat | 30 +++++++++++++++++++++++++----- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 9b3d236121..bc1fdc39c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disallow\sthe\suse\sof\sCOLLATE\sclauses\sand\sthe\sASC\sand\sDESC\skeywords\swithin\nforeign\skey\sconstraints\sand\sin\sthe\sargument\slist\sto\scommon\stable\sexpressions. -D 2015-08-24T15:39:42.405 +C Enhancements\sto\sthe\sbatch\sbuild\stool\sfor\sMSVC. +D 2015-08-24T17:18:43.705 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1323,7 +1323,7 @@ F test/without_rowid6.test db0dbf03c49030aa3c1ba5f618620334bd2baf5f F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac -F tool/build-all-msvc.bat 60dbf6021d3de0a98575f6dfe4e12bd80b3edcf0 x +F tool/build-all-msvc.bat 761d8c82a1a529261291812732a853a1b4256d85 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -1379,7 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 196d66d34d9783622e6f2f79eafea1488fc6f5cf -R 2a74528f51d8172b5387a3b7f110bb2b -U drh -Z ffc4f8e604c67b08cae8ed1eda808a43 +P 83cbc4d8761498647794affffa961a4fca311be7 +R 854ee56e3c07dedcc4aa69e0f36f6060 +U mistachkin +Z 2aa32a441452e19f6c0e00af3c0b6972 diff --git a/manifest.uuid b/manifest.uuid index 9ca8f1fa75..fbc4c6a88f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83cbc4d8761498647794affffa961a4fca311be7 \ No newline at end of file +a1ae20cd97456a1126cfa1a9bedce0bac0940ad6 \ No newline at end of file diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index 25435838a3..4842dc4074 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -312,12 +312,26 @@ IF "%VisualStudioVersion%" == "11.0" ( ) ) +REM +REM NOTE: This is the name of the sub-directory where the UCRT libraries may +REM be found. It is only used when compiling against the UCRT. +REM +IF DEFINED UCRTVersion ( + SET NUCRTVER=%UCRTVersion% +) ELSE ( + SET NUCRTVER=10.0.10240.0 +) + REM REM NOTE: This is the name of the sub-directory where the Windows 10.0 SDK REM libraries may be found. It is only used when compiling with the REM Windows 10.0 SDK. REM -SET WIN10LIBDIR=10.0.10240.0 +IF DEFINED WindowsSDKLibVersion ( + SET WIN10SDKVER=%WindowsSDKLibVersion:\=% +) ELSE ( + SET WIN10SDKVER=%NUCRTVER% +) REM REM NOTE: Check if this is the Windows Phone SDK. If so, a different batch @@ -361,6 +375,7 @@ FOR %%P IN (%PLATFORMS%) DO ( REM CALL :fn_UnsetVariable CommandPromptType CALL :fn_UnsetVariable DevEnvDir + CALL :fn_UnsetVariable DNX_HOME CALL :fn_UnsetVariable ExtensionSdkDir CALL :fn_UnsetVariable Framework35Version CALL :fn_UnsetVariable Framework40Version @@ -372,14 +387,19 @@ FOR %%P IN (%PLATFORMS%) DO ( CALL :fn_UnsetVariable INCLUDE CALL :fn_UnsetVariable LIB CALL :fn_UnsetVariable LIBPATH + CALL :fn_UnsetVariable NETFXSDKDir CALL :fn_UnsetVariable Platform + CALL :fn_UnsetVariable UCRTVersion CALL :fn_UnsetVariable UniversalCRTSdkDir REM CALL :fn_UnsetVariable VCINSTALLDIR CALL :fn_UnsetVariable VSINSTALLDIR + CALL :fn_UnsetVariable WindowsLibPath CALL :fn_UnsetVariable WindowsPhoneKitDir CALL :fn_UnsetVariable WindowsSdkDir CALL :fn_UnsetVariable WindowsSdkDir_35 CALL :fn_UnsetVariable WindowsSdkDir_old + CALL :fn_UnsetVariable WindowsSDKLibVersion + CALL :fn_UnsetVariable WindowsSDKVersion CALL :fn_UnsetVariable WindowsSDK_ExecutablePath_x86 CALL :fn_UnsetVariable WindowsSDK_ExecutablePath_x64 @@ -489,9 +509,9 @@ FOR %%P IN (%PLATFORMS%) DO ( REM different directory naming conventions. REM IF DEFINED USE_WINV100_NSDKLIBPATH ( - CALL :fn_AppendVariable NSDKLIBPATH \..\10\lib\%WIN10LIBDIR%\um\x86 - CALL :fn_CopyVariable UniversalCRTSdkDir PSDKLIBPATH - CALL :fn_AppendVariable PSDKLIBPATH Lib\%WIN10LIBDIR%\um\%%D + CALL :fn_AppendVariable NSDKLIBPATH \..\10\lib\%WIN10SDKVER%\um\x86 + CALL :fn_CopyVariable WindowsSdkDir PSDKLIBPATH + CALL :fn_AppendVariable PSDKLIBPATH lib\%WIN10SDKVER%\um\%%D ) ELSE IF DEFINED USE_WINV63_NSDKLIBPATH ( CALL :fn_AppendVariable NSDKLIBPATH \lib\winv6.3\um\x86 ) ELSE IF "%VisualStudioVersion%" == "12.0" ( @@ -514,7 +534,7 @@ FOR %%P IN (%PLATFORMS%) DO ( IF DEFINED SET_NUCRTLIBPATH ( IF DEFINED UniversalCRTSdkDir ( CALL :fn_CopyVariable UniversalCRTSdkDir NUCRTLIBPATH - CALL :fn_AppendVariable NUCRTLIBPATH \lib\%WIN10LIBDIR%\ucrt\x86 + CALL :fn_AppendVariable NUCRTLIBPATH \lib\%NUCRTVER%\ucrt\x86 ) ) From 8981b904b545ad056ad2f6eb0ebef6a6b9606b5b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Aug 2015 17:42:49 +0000 Subject: [PATCH 34/54] Enhance the CREATE VIEW syntax so that the names of columns of the view can be specified after the view name. FossilOrigin-Name: d794b34da6f9c77dfe17773b0b17b22de72cce7f --- manifest | 22 ++++++------ manifest.uuid | 2 +- src/build.c | 89 ++++++++++++++++++++++++++----------------------- src/parse.y | 19 ++++++----- src/select.c | 8 ++--- src/sqliteInt.h | 6 ++-- test/view.test | 11 +++++- 7 files changed, 87 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index bc1fdc39c8..ac41156aae 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhancements\sto\sthe\sbatch\sbuild\stool\sfor\sMSVC. -D 2015-08-24T17:18:43.705 +C Enhance\sthe\sCREATE\sVIEW\ssyntax\sso\sthat\sthe\snames\sof\scolumns\sof\sthe\sview\scan\nbe\sspecified\safter\sthe\sview\sname. +D 2015-08-24T17:42:49.622 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c f49c55c1fba430c2d6af5039d0bf14de5ae6a427 +F src/build.c a4e2669bc59729589b7e3867c58eae5a4f2419ff F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -326,7 +326,7 @@ F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c aa916ca28606ccf4b6877dfc2b643ccbca86589f F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 -F src/parse.y b5e0a5f8bb9ec33a58aa34850b6fae46aac51fdd +F src/parse.y 07f2084f9ec157b108f1bf12466277d6f17b59d1 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c a3fe31b17e841ec70beee72a2c960e9c787a8857 @@ -337,12 +337,12 @@ F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 7a67cd2aebc9a9eeecd1d104eb6a9237388eb452 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 24323faace224dce1e6b65276605de1db39d77a4 +F src/select.c da6d1e7a4f1c8d713ed5415b5ed21d82ef465c0f F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h 1741691824491c330dda1e49f370920bfd3aa230 +F src/sqliteInt.h 79a8e76bcbe67170d371ae2a08c85cc2d7cd0caf F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -1239,7 +1239,7 @@ F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 -F test/view.test f44014f78d7650fb4bfb8ef96a5e4dc8f25eb083 +F test/view.test 3930ae94042d702ab15a6a0ef692cfa5c9f9b68b F test/vtab1.test 6210e076997f176bedc300a87ad6404651b601dd F test/vtab2.test f8cd1bb9aba7143eba97812d9617880a36d247ad F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -1379,7 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 83cbc4d8761498647794affffa961a4fca311be7 -R 854ee56e3c07dedcc4aa69e0f36f6060 -U mistachkin -Z 2aa32a441452e19f6c0e00af3c0b6972 +P a1ae20cd97456a1126cfa1a9bedce0bac0940ad6 +R 7c6b5f2ebead0dd2f6f395aca9a96d64 +U drh +Z 9176898b8b319f8964514d527c2d8cd5 diff --git a/manifest.uuid b/manifest.uuid index fbc4c6a88f..358a0a2590 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1ae20cd97456a1126cfa1a9bedce0bac0940ad6 \ No newline at end of file +d794b34da6f9c77dfe17773b0b17b22de72cce7f \ No newline at end of file diff --git a/src/build.c b/src/build.c index 0a816b398e..815b17deb8 100644 --- a/src/build.c +++ b/src/build.c @@ -2056,6 +2056,7 @@ void sqlite3CreateView( Token *pBegin, /* The CREATE token that begins the statement */ Token *pName1, /* The token that holds the name of the view */ Token *pName2, /* The token that holds the name of the view */ + ExprList *pCNames, /* Optional list of view column names */ Select *pSelect, /* A SELECT statement that will become the new view */ int isTemp, /* TRUE for a TEMPORARY view */ int noErr /* Suppress error messages if VIEW already exists */ @@ -2075,18 +2076,13 @@ void sqlite3CreateView( return; } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); + sqlite3RestrictColumnListSyntax(pParse, pCNames); p = pParse->pNewTable; - if( p==0 || pParse->nErr ){ - sqlite3SelectDelete(db, pSelect); - return; - } + if( p==0 || pParse->nErr ) goto create_view_fail; sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); - if( sqlite3FixSelect(&sFix, pSelect) ){ - sqlite3SelectDelete(db, pSelect); - return; - } + if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; /* Make a copy of the entire SELECT statement that defines the view. ** This will force all the Expr.token.z values to be dynamically @@ -2094,27 +2090,31 @@ void sqlite3CreateView( ** they will persist after the current sqlite3_exec() call returns. */ p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); - sqlite3SelectDelete(db, pSelect); - if( db->mallocFailed ){ - return; - } + p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); + if( db->mallocFailed ) goto create_view_fail; /* Locate the end of the CREATE VIEW statement. Make sEnd point to ** the end. */ sEnd = pParse->sLastToken; - if( ALWAYS(sEnd.z[0]!=0) && sEnd.z[0]!=';' ){ + assert( sEnd.z[0]!=0 ); + if( sEnd.z[0]!=';' ){ sEnd.z += sEnd.n; } sEnd.n = 0; n = (int)(sEnd.z - pBegin->z); + assert( n>0 ); z = pBegin->z; - while( ALWAYS(n>0) && sqlite3Isspace(z[n-1]) ){ n--; } + while( sqlite3Isspace(z[n-1]) ){ n--; } sEnd.z = &z[n-1]; sEnd.n = 1; /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ sqlite3EndTable(pParse, 0, &sEnd, 0, 0); + +create_view_fail: + sqlite3SelectDelete(db, pSelect); + sqlite3ExprListDelete(db, pCNames); return; } #endif /* SQLITE_OMIT_VIEW */ @@ -2132,6 +2132,7 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ sqlite3_xauth xAuth; /* Saved xAuth pointer */ + u8 bEnabledLA; /* Saved db->lookaside.bEnabled state */ assert( pTable ); @@ -2177,40 +2178,46 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ ** statement that defines the view. */ assert( pTable->pSelect ); - pSel = sqlite3SelectDup(db, pTable->pSelect, 0); - if( pSel ){ - u8 enableLookaside = db->lookaside.bEnabled; - n = pParse->nTab; - sqlite3SrcListAssignCursors(pParse, pSel->pSrc); - pTable->nCol = -1; + bEnabledLA = db->lookaside.bEnabled; + if( pTable->pCheck ){ db->lookaside.bEnabled = 0; + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + &pTable->nCol, &pTable->aCol); + }else{ + pSel = sqlite3SelectDup(db, pTable->pSelect, 0); + if( pSel ){ + n = pParse->nTab; + sqlite3SrcListAssignCursors(pParse, pSel->pSrc); + pTable->nCol = -1; + db->lookaside.bEnabled = 0; #ifndef SQLITE_OMIT_AUTHORIZATION - xAuth = db->xAuth; - db->xAuth = 0; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); - db->xAuth = xAuth; + xAuth = db->xAuth; + db->xAuth = 0; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + db->xAuth = xAuth; #else - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); #endif - db->lookaside.bEnabled = enableLookaside; - pParse->nTab = n; - if( pSelTab ){ - assert( pTable->aCol==0 ); - pTable->nCol = pSelTab->nCol; - pTable->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - pTable->pSchema->schemaFlags |= DB_UnresetViews; - }else{ - pTable->nCol = 0; + pParse->nTab = n; + if( pSelTab ){ + assert( pTable->aCol==0 ); + pTable->nCol = pSelTab->nCol; + pTable->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); + }else{ + pTable->nCol = 0; + nErr++; + } + sqlite3SelectDelete(db, pSel); + } else { nErr++; } - sqlite3SelectDelete(db, pSel); - } else { - nErr++; } + db->lookaside.bEnabled = bEnabledLA; + pTable->pSchema->schemaFlags |= DB_UnresetViews; #endif /* SQLITE_OMIT_VIEW */ return nErr; } diff --git a/src/parse.y b/src/parse.y index bc193ec1f2..acd899449e 100644 --- a/src/parse.y +++ b/src/parse.y @@ -386,8 +386,9 @@ ifexists(A) ::= . {A = 0;} ///////////////////// The CREATE VIEW statement ///////////////////////////// // %ifndef SQLITE_OMIT_VIEW -cmd ::= createkw(X) temp(T) VIEW ifnotexists(E) nm(Y) dbnm(Z) AS select(S). { - sqlite3CreateView(pParse, &X, &Y, &Z, S, T, E); +cmd ::= createkw(X) temp(T) VIEW ifnotexists(E) nm(Y) dbnm(Z) idxlist_opt(C) + AS select(S). { + sqlite3CreateView(pParse, &X, &Y, &Z, C, S, T, E); } cmd ::= DROP VIEW ifexists(E) fullname(X). { sqlite3DropTable(pParse, X, 1, E); @@ -784,11 +785,11 @@ setlist(A) ::= nm(X) EQ expr(Y). { ////////////////////////// The INSERT command ///////////////////////////////// // -cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). { +cmd ::= with(W) insert_cmd(R) INTO fullname(X) idlist_opt(F) select(S). { sqlite3WithPush(pParse, W, 1); sqlite3Insert(pParse, X, S, F, R); } -cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. +cmd ::= with(W) insert_cmd(R) INTO fullname(X) idlist_opt(F) DEFAULT VALUES. { sqlite3WithPush(pParse, W, 1); sqlite3Insert(pParse, X, 0, F, R); @@ -798,13 +799,13 @@ cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. insert_cmd(A) ::= INSERT orconf(R). {A = R;} insert_cmd(A) ::= REPLACE. {A = OE_Replace;} -%type inscollist_opt {IdList*} -%destructor inscollist_opt {sqlite3IdListDelete(pParse->db, $$);} +%type idlist_opt {IdList*} +%destructor idlist_opt {sqlite3IdListDelete(pParse->db, $$);} %type idlist {IdList*} %destructor idlist {sqlite3IdListDelete(pParse->db, $$);} -inscollist_opt(A) ::= . {A = 0;} -inscollist_opt(A) ::= LP idlist(X) RP. {A = X;} +idlist_opt(A) ::= . {A = 0;} +idlist_opt(A) ::= LP idlist(X) RP. {A = X;} idlist(A) ::= idlist(X) COMMA nm(Y). {A = sqlite3IdListAppend(pParse->db,X,&Y);} idlist(A) ::= nm(Y). @@ -1369,7 +1370,7 @@ trigger_cmd(A) ::= { A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R); } // INSERT -trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) inscollist_opt(F) select(S). +trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) idlist_opt(F) select(S). {A = sqlite3TriggerInsertStep(pParse->db, &X, F, S, R);} // DELETE diff --git a/src/select.c b/src/select.c index e767111d8d..2581a240e9 100644 --- a/src/select.c +++ b/src/select.c @@ -1587,7 +1587,7 @@ static void generateColumnNames( ** Return SQLITE_OK on success. If a memory allocation error occurs, ** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. */ -static int selectColumnsFromExprList( +int sqlite3ColumnsFromExprList( Parse *pParse, /* Parsing context */ ExprList *pEList, /* Expr list from which to derive column names */ i16 *pnCol, /* Write the number of columns here */ @@ -1754,7 +1754,7 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ pTab->nRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ @@ -4121,7 +4121,7 @@ static int withExpand( pEList = pCte->pCols; } - selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); if( bMayRecursive ){ if( pSel->selFlags & SF_Recursive ){ pCte->zErr = "multiple recursive references: %s"; @@ -4244,7 +4244,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } - selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); + sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 18a0fd705f..5f12350538 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1632,9 +1632,8 @@ struct Table { Select *pSelect; /* NULL for tables. Points to definition if a view. */ FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ -#ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ -#endif + /* ... also used as column name list in a VIEW */ int tnum; /* Root BTree page for this table */ i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ @@ -3260,6 +3259,7 @@ void sqlite3CollapseDatabaseArray(sqlite3*); void sqlite3BeginParse(Parse*,int); void sqlite3CommitInternalChanges(sqlite3*); void sqlite3DeleteColumnNames(sqlite3*,Table*); +int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); Table *sqlite3ResultSetOfSelect(Parse*,Select*); void sqlite3OpenMasterTable(Parse *, int); Index *sqlite3PrimaryKeyIndex(Table*); @@ -3301,7 +3301,7 @@ void sqlite3RowSetInsert(RowSet*, i64); int sqlite3RowSetTest(RowSet*, int iBatch, i64); int sqlite3RowSetNext(RowSet*, i64*); -void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); +void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int); #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) int sqlite3ViewGetColumnNames(Parse*,Table*); diff --git a/test/view.test b/test/view.test index a5fe85e648..b39487689c 100644 --- a/test/view.test +++ b/test/view.test @@ -152,6 +152,15 @@ do_test view-3.3.2 { SELECT * FROM v1b LIMIT 1 } } {a 2 b+c 7 c 4} +do_test view-3.3.3 { + execsql2 { + CREATE VIEW v1c(x,y,z) AS SELECT a, b+c, c-b FROM t1; + SELECT * FROM v1c LIMIT 1; + } +} {x 2 y 7 z 1} +do_catchsql_test view-3.3.4 { + CREATE VIEW v1err(x,y DESC,z) AS SELECT a, b+c, c-b FROM t1; +} {1 {syntax error after column name "y"}} ifcapable compound { do_test view-3.4 { @@ -337,7 +346,7 @@ do_test view-8.2 { } {7 2 13 5 19 8 27 12} do_test view-8.3 { execsql { - CREATE VIEW v7 AS SELECT pqr+xyz AS a FROM v6; + CREATE VIEW v7(a) AS SELECT pqr+xyz FROM v6; SELECT * FROM v7 ORDER BY a; } } {9 18 27 39} From 108aa00a87e19312fa04b9cbaf98c8587128ab86 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Aug 2015 20:21:20 +0000 Subject: [PATCH 35/54] Enhances the parser so that it accepts arbitrary expressions for the arguments of an index, though the code generator still rejects everything other than simple column names. The sqlite3RestrictColumnListSyntax() routine is removed since that feature is now handled by the parser. FossilOrigin-Name: bed42116addabcf3dfdc2e2d51ae183965704988 --- manifest | 22 +++++------ manifest.uuid | 2 +- src/build.c | 78 +++++++++++++++---------------------- src/expr.c | 1 - src/parse.y | 98 ++++++++++++++++++++++++++++++++--------------- src/sqliteInt.h | 2 - test/parser1.test | 23 ++++++++++- test/with2.test | 4 +- 8 files changed, 135 insertions(+), 95 deletions(-) diff --git a/manifest b/manifest index ac41156aae..bc354b2e92 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sCREATE\sVIEW\ssyntax\sso\sthat\sthe\snames\sof\scolumns\sof\sthe\sview\scan\nbe\sspecified\safter\sthe\sview\sname. -D 2015-08-24T17:42:49.622 +C Enhances\sthe\sparser\sso\sthat\sit\saccepts\sarbitrary\sexpressions\sfor\sthe\sarguments\nof\san\sindex,\sthough\sthe\scode\sgenerator\sstill\srejects\severything\sother\sthan\nsimple\scolumn\snames.\s\sThe\ssqlite3RestrictColumnListSyntax()\sroutine\sis\sremoved\nsince\sthat\sfeature\sis\snow\shandled\sby\sthe\sparser. +D 2015-08-24T20:21:20.985 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,14 +282,14 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c a4e2669bc59729589b7e3867c58eae5a4f2419ff +F src/build.c 789e75f3478ac63c0f398a131c49a0802c356c2b F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c 8ec787fed4929d8ccdf6b1bc360fccc3e1d2ca58 F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a F src/delete.c 8857a6f27560718f65d43bdbec86c967ae1f8dfa -F src/expr.c c05d67f1a03c097d5c29839d5a538cfde9c472ce +F src/expr.c 650ac7c4f659980a3315e2aaa02a0d71e87f14a5 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c c9b63a217d86582c22121699a47f22f524608869 F src/func.c 824bea430d3a2b7dbc62806ad54da8fdb8ed9e3f @@ -326,7 +326,7 @@ F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c aa916ca28606ccf4b6877dfc2b643ccbca86589f F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 -F src/parse.y 07f2084f9ec157b108f1bf12466277d6f17b59d1 +F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c a3fe31b17e841ec70beee72a2c960e9c787a8857 @@ -342,7 +342,7 @@ F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h 79a8e76bcbe67170d371ae2a08c85cc2d7cd0caf +F src/sqliteInt.h edbcd0c0787541a636a25ab1d1eaf847dbd043f1 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -910,7 +910,7 @@ F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 -F test/parser1.test 23867b6f2c4758c7774108826d9f17e9cd17bcde +F test/parser1.test 222b5cbf3e2e659fec1bf7d723488c8b9c94f1d0 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff @@ -1312,7 +1312,7 @@ F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test a1e8660be88e2eb4648f8860f831d1e38b5b5443 -F test/with2.test ee227a663586aa09771cafd4fa269c5217eaf775 +F test/with2.test 2b40da883658eb74ad8ad06afabe11a408e7fb87 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test 1a7b9bd51b899928d327052df9741d2fe8dbe701 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1379,7 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a1ae20cd97456a1126cfa1a9bedce0bac0940ad6 -R 7c6b5f2ebead0dd2f6f395aca9a96d64 +P d794b34da6f9c77dfe17773b0b17b22de72cce7f +R 551ba7ed9667cd3021493a6e99ba80fb U drh -Z 9176898b8b319f8964514d527c2d8cd5 +Z 12005a1b68dc81ba8d64d2d3b63a147c diff --git a/manifest.uuid b/manifest.uuid index 358a0a2590..defc8a5329 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d794b34da6f9c77dfe17773b0b17b22de72cce7f \ No newline at end of file +bed42116addabcf3dfdc2e2d51ae183965704988 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 815b17deb8..8a7dda89c1 100644 --- a/src/build.c +++ b/src/build.c @@ -1310,11 +1310,15 @@ void sqlite3AddPrimaryKey( }else{ nTerm = pList->nExpr; for(i=0; inCol; iCol++){ - if( sqlite3StrICmp(pList->a[i].zName, pTab->aCol[iCol].zName)==0 ){ - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zType = pTab->aCol[iCol].zType; - break; + Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[i].pExpr); + if( pCExpr && pCExpr->op==TK_ID ){ + const char *zCName = pCExpr->u.zToken; + for(iCol=0; iColnCol; iCol++){ + if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ + pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; + zType = pTab->aCol[iCol].zType; + break; + } } } } @@ -1696,10 +1700,12 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ */ if( pTab->iPKey>=0 ){ ExprList *pList; - pList = sqlite3ExprListAppend(pParse, 0, 0); + Token ipkToken; + ipkToken.z = pTab->aCol[pTab->iPKey].zName; + ipkToken.n = sqlite3Strlen30(ipkToken.z); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); if( pList==0 ) return; - pList->a[0].zName = sqlite3DbStrDup(pParse->db, - pTab->aCol[pTab->iPKey].zName); pList->a[0].sortOrder = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); @@ -2076,7 +2082,6 @@ void sqlite3CreateView( return; } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); - sqlite3RestrictColumnListSyntax(pParse, pCNames); p = pParse->pNewTable; if( p==0 || pParse->nErr ) goto create_view_fail; sqlite3TwoPartName(pParse, pName1, pName2, &pName); @@ -2607,8 +2612,6 @@ void sqlite3CreateForeignKey( assert( pTo!=0 ); if( p==0 || IN_DECLARE_VTAB ) goto fk_end; - sqlite3RestrictColumnListSyntax(pParse, pFromCol); - sqlite3RestrictColumnListSyntax(pParse, pToCol); if( pFromCol==0 ){ int iCol = p->nCol-1; if( NEVER(iCol<0) ) goto fk_end; @@ -3043,12 +3046,16 @@ Index *sqlite3CreateIndex( ** So create a fake list to simulate this. */ if( pList==0 ){ - pList = sqlite3ExprListAppend(pParse, 0, 0); + Token prevCol; + prevCol.z = pTab->aCol[pTab->nCol-1].zName; + prevCol.n = sqlite3Strlen30(prevCol.z); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; - pList->a[0].zName = sqlite3DbStrDup(pParse->db, - pTab->aCol[pTab->nCol-1].zName); assert( pList->nExpr==1 ); sqlite3ExprListSetSortOrder(pList, sortOrder); + }else{ + sqlite3ExprListCheckLength(pParse, pList, "index"); } /* Figure out how many bytes of space are required to store explicitly @@ -3056,8 +3063,7 @@ Index *sqlite3CreateIndex( */ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( pExpr ){ - assert( pExpr->op==TK_COLLATE ); + if( pExpr && pExpr->op==TK_COLLATE ){ nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -3109,10 +3115,17 @@ Index *sqlite3CreateIndex( ** break backwards compatibility - it needs to be a warning. */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ - const char *zColName = pListItem->zName; + const char *zColName; + Expr *pCExpr; int requestedSortOrder; char *zColl; /* Collation sequence name */ + pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); + if( pCExpr->op!=TK_ID ){ + sqlite3ErrorMsg(pParse, "indexes on expressions not yet supported"); + continue; + } + zColName = pCExpr->u.zToken; for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; } @@ -3124,9 +3137,8 @@ Index *sqlite3CreateIndex( } assert( j<=0x7fff ); pIndex->aiColumn[i] = (i16)j; - if( pListItem->pExpr ){ + if( pListItem->pExpr->op==TK_COLLATE ){ int nColl; - assert( pListItem->pExpr->op==TK_COLLATE ); zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); @@ -4293,32 +4305,6 @@ KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ return pKey; } -/* -** Generate a syntax error if the expression list provided contains -** any COLLATE or ASC or DESC keywords. -** -** Some legacy versions of SQLite allowed constructs like: -** -** CREATE TABLE x(..., FOREIGN KEY(x COLLATE binary DESC) REFERENCES...); -** ^^^^^^^^^^^^^^^^^^^ -** -** The COLLATE and sort order terms were ignored. To prevent compatibility -** problems in case something like this appears in a legacy sqlite_master -** table, only enforce the restriction on new SQL statements, not when -** parsing the schema out of the sqlite_master table. -*/ -void sqlite3RestrictColumnListSyntax(Parse *pParse, ExprList *p){ - int i; - if( p==0 || pParse->db->init.busy ) return; - for(i=0; inExpr; i++){ - if( p->a[i].pExpr!=0 || p->a[i].bDefinedSO ){ - sqlite3ErrorMsg(pParse, "syntax error after column name \"%w\"", - p->a[i].zName); - return; - } - } -} - #ifndef SQLITE_OMIT_CTE /* ** This routine is invoked once per CTE by the parser while parsing a @@ -4335,8 +4321,6 @@ With *sqlite3WithAdd( With *pNew; char *zName; - sqlite3RestrictColumnListSyntax(pParse, pArglist); - /* Check that the CTE name is unique within this WITH clause. If ** not, store an error in the Parse structure. */ zName = sqlite3NameFromToken(pParse->db, pName); diff --git a/src/expr.c b/src/expr.c index 1c57ecc6fd..1aebef6b16 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1172,7 +1172,6 @@ void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ return; } p->a[p->nExpr-1].sortOrder = (u8)iSortOrder; - p->a[p->nExpr-1].bDefinedSO = 1; } /* diff --git a/src/parse.y b/src/parse.y index acd899449e..e99feeefc1 100644 --- a/src/parse.y +++ b/src/parse.y @@ -301,7 +301,7 @@ ccons ::= PRIMARY KEY sortorder(Z) onconf(R) autoinc(I). {sqlite3AddPrimaryKey(pParse,0,R,I,Z);} ccons ::= UNIQUE onconf(R). {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0);} ccons ::= CHECK LP expr(X) RP. {sqlite3AddCheckConstraint(pParse,X.pExpr);} -ccons ::= REFERENCES nm(T) idxlist_opt(TA) refargs(R). +ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R). {sqlite3CreateForeignKey(pParse,0,&T,TA,R);} ccons ::= defer_subclause(D). {sqlite3DeferForeignKey(pParse,D);} ccons ::= COLLATE ids(C). {sqlite3AddCollateType(pParse, &C);} @@ -345,14 +345,14 @@ conslist ::= tcons. tconscomma ::= COMMA. {pParse->constraintName.n = 0;} tconscomma ::= . tcons ::= CONSTRAINT nm(X). {pParse->constraintName = X;} -tcons ::= PRIMARY KEY LP idxlist(X) autoinc(I) RP onconf(R). +tcons ::= PRIMARY KEY LP sortlist(X) autoinc(I) RP onconf(R). {sqlite3AddPrimaryKey(pParse,X,R,I,0);} -tcons ::= UNIQUE LP idxlist(X) RP onconf(R). +tcons ::= UNIQUE LP sortlist(X) RP onconf(R). {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0);} tcons ::= CHECK LP expr(E) RP onconf. {sqlite3AddCheckConstraint(pParse,E.pExpr);} -tcons ::= FOREIGN KEY LP idxlist(FA) RP - REFERENCES nm(T) idxlist_opt(TA) refargs(R) defer_subclause_opt(D). { +tcons ::= FOREIGN KEY LP eidlist(FA) RP + REFERENCES nm(T) eidlist_opt(TA) refargs(R) defer_subclause_opt(D). { sqlite3CreateForeignKey(pParse, FA, &T, TA, R); sqlite3DeferForeignKey(pParse, D); } @@ -386,7 +386,7 @@ ifexists(A) ::= . {A = 0;} ///////////////////// The CREATE VIEW statement ///////////////////////////// // %ifndef SQLITE_OMIT_VIEW -cmd ::= createkw(X) temp(T) VIEW ifnotexists(E) nm(Y) dbnm(Z) idxlist_opt(C) +cmd ::= createkw(X) temp(T) VIEW ifnotexists(E) nm(Y) dbnm(Z) eidlist_opt(C) AS select(S). { sqlite3CreateView(pParse, &X, &Y, &Z, C, S, T, E); } @@ -674,6 +674,11 @@ using_opt(U) ::= . {U = 0;} %type orderby_opt {ExprList*} %destructor orderby_opt {sqlite3ExprListDelete(pParse->db, $$);} + +// the sortlist non-terminal stores a list of expression where each +// expression is optionally followed by ASC or DESC to indicate the +// sort order. +// %type sortlist {ExprList*} %destructor sortlist {sqlite3ExprListDelete(pParse->db, $$);} @@ -1208,7 +1213,7 @@ nexprlist(A) ::= expr(Y). ///////////////////////////// The CREATE INDEX command /////////////////////// // cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D) - ON nm(Y) LP idxlist(Z) RP where_opt(W). { + ON nm(Y) LP sortlist(Z) RP where_opt(W). { sqlite3CreateIndex(pParse, &X, &D, sqlite3SrcListAppend(pParse->db,0,&Y,0), Z, U, &S, W, SQLITE_SO_ASC, NE); @@ -1218,31 +1223,64 @@ cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D) uniqueflag(A) ::= UNIQUE. {A = OE_Abort;} uniqueflag(A) ::= . {A = OE_None;} -%type idxlist {ExprList*} -%destructor idxlist {sqlite3ExprListDelete(pParse->db, $$);} -%type idxlist_opt {ExprList*} -%destructor idxlist_opt {sqlite3ExprListDelete(pParse->db, $$);} -idxlist_opt(A) ::= . {A = 0;} -idxlist_opt(A) ::= LP idxlist(X) RP. {A = X;} -idxlist(A) ::= idxlist(X) COMMA nm(Y) collate(C) sortorder(Z). { - Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C, 1); - A = sqlite3ExprListAppend(pParse,X, p); - sqlite3ExprListSetName(pParse,A,&Y,1); - sqlite3ExprListCheckLength(pParse, A, "index"); - sqlite3ExprListSetSortOrder(A,Z); +// The eidlist non-terminal (Expression Id List) generates an ExprList +// from a list of identifiers. The identifier names are in ExprList.a[].zName. +// This list is stored in an ExprList rather than an IdList so that it +// can be easily sent to sqlite3ColumnsExprList(). +// +// eidlist is grouped with CREATE INDEX because it used to be the non-terminal +// used for the arguments to an index. That is just an historical accident. +// +// IMPORTANT COMPATIBILITY NOTE: Some prior versions of SQLite accepted +// COLLATE clauses and ASC or DESC keywords on ID lists in inappropriate +// places - places that might have been stored in the sqlite_master schema. +// Those extra features were ignored. But because they might be in some +// (busted) old databases, we need to continue parsing them when loading +// historical schemas. +// +%type eidlist {ExprList*} +%destructor eidlist {sqlite3ExprListDelete(pParse->db, $$);} +%type eidlist_opt {ExprList*} +%destructor eidlist_opt {sqlite3ExprListDelete(pParse->db, $$);} + +%include { + /* Add a single new term to an ExprList that is used to store a + ** list of identifiers. Report an error if the ID list contains + ** a COLLATE clause or an ASC or DESC keyword, except ignore the + ** error while parsing a legacy schema. + */ + static ExprList *parserAddExprIdListTerm( + Parse *pParse, + ExprList *pPrior, + Token *pIdToken, + int hasCollate, + int sortOrder + ){ + ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0); + if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED) + && pParse->db->init.busy==0 + ){ + sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"", + pIdToken->n, pIdToken->z); + } + sqlite3ExprListSetName(pParse, p, pIdToken, 1); + return p; + } +} // end %include + +eidlist_opt(A) ::= . {A = 0;} +eidlist_opt(A) ::= LP eidlist(X) RP. {A = X;} +eidlist(A) ::= eidlist(X) COMMA nm(Y) collate(C) sortorder(Z). { + A = parserAddExprIdListTerm(pParse, X, &Y, C, Z); } -idxlist(A) ::= nm(Y) collate(C) sortorder(Z). { - Expr *p = sqlite3ExprAddCollateToken(pParse, 0, &C, 1); - A = sqlite3ExprListAppend(pParse,0, p); - sqlite3ExprListSetName(pParse, A, &Y, 1); - sqlite3ExprListCheckLength(pParse, A, "index"); - sqlite3ExprListSetSortOrder(A,Z); +eidlist(A) ::= nm(Y) collate(C) sortorder(Z). { + A = parserAddExprIdListTerm(pParse, 0, &Y, C, Z); } -%type collate {Token} -collate(C) ::= . {C.z = 0; C.n = 0;} -collate(C) ::= COLLATE ids(X). {C = X;} +%type collate {int} +collate(C) ::= . {C = 0;} +collate(C) ::= COLLATE ids. {C = 1;} ///////////////////////////// The DROP INDEX command ///////////////////////// @@ -1490,10 +1528,10 @@ with(A) ::= . {A = 0;} with(A) ::= WITH wqlist(W). { A = W; } with(A) ::= WITH RECURSIVE wqlist(W). { A = W; } -wqlist(A) ::= nm(X) idxlist_opt(Y) AS LP select(Z) RP. { +wqlist(A) ::= nm(X) eidlist_opt(Y) AS LP select(Z) RP. { A = sqlite3WithAdd(pParse, 0, &X, Y, Z); } -wqlist(A) ::= wqlist(W) COMMA nm(X) idxlist_opt(Y) AS LP select(Z) RP. { +wqlist(A) ::= wqlist(W) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. { A = sqlite3WithAdd(pParse, W, &X, Y, Z); } %endif SQLITE_OMIT_CTE diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5f12350538..4c17904ff8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2189,7 +2189,6 @@ struct ExprList { unsigned done :1; /* A flag to indicate when processing is finished */ unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ unsigned reusable :1; /* Constant expression is reusable */ - unsigned bDefinedSO :1; /* True if either DESC or ASC keywords present */ union { struct { u16 iOrderByCol; /* For ORDER BY, column number in result set */ @@ -3758,7 +3757,6 @@ const char *sqlite3JournalModename(int); int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #endif -void sqlite3RestrictColumnListSyntax(Parse*,ExprList*); #ifndef SQLITE_OMIT_CTE With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*); void sqlite3WithDelete(sqlite3*,With*); diff --git a/test/parser1.test b/test/parser1.test index f4c3227f87..78c1a40c63 100644 --- a/test/parser1.test +++ b/test/parser1.test @@ -21,7 +21,13 @@ do_catchsql_test parser1-1.1 { b TEXT, FOREIGN KEY(b COLLATE nocase DESC) REFERENCES t1(a COLLATE binary ASC) ); -} {1 {syntax error after column name "a"}} +} {1 {syntax error after column name "b"}} + + +# Verify that a legacy schema in the sqlite_master file is allowed to have +# COLLATE, ASC, and DESC keywords on the id list of a FK constraint, and that +# those keywords are silently ignored. +# do_execsql_test parser1-1.2 { CREATE TABLE t1( a TEXT PRIMARY KEY, @@ -42,7 +48,22 @@ do_test parser1-1.3 { sqlite3 db2 test.db db2 eval {SELECT * FROM t1 ORDER BY 1} } {abc {} xyz abc} +db2 close +do_execsql_test parser1-1.4 { + UPDATE sqlite_master SET sql='CREATE TABLE t1( + a TEXT PRIMARY KEY, + b TEXT, + FOREIGN KEY(b ASC) REFERENCES t1(a) + )' WHERE name='t1'; + SELECT name FROM sqlite_master WHERE sql LIKE '%ASC%'; +} {t1} +sqlite3 db2 test.db +do_test parser1-1.5 { + sqlite3 db2 test.db + db2 eval {SELECT * FROM t1 ORDER BY 1} +} {abc {} xyz abc} +db2 close do_catchsql_test parser1-2.1 { WITH RECURSIVE diff --git a/test/with2.test b/test/with2.test index eb0614729b..02d10b5112 100644 --- a/test/with2.test +++ b/test/with2.test @@ -250,7 +250,8 @@ do_execsql_test 4.4 [genstmt 255] 1 set nLimit [sqlite3_limit db SQLITE_LIMIT_COLUMN -1] do_execsql_test 4.5 [genstmt [expr $nLimit-1]] 1 do_execsql_test 4.6 [genstmt $nLimit] 1 -do_catchsql_test 4.7 [genstmt [expr $nLimit+1]] {1 {too many columns in index}} +do_catchsql_test 4.7 [genstmt [expr $nLimit+1]] \ + {1 {too many columns in result set}} #--------------------------------------------------------------------------- # Check that adding a WITH clause to an INSERT disables the xfer @@ -415,4 +416,3 @@ do_execsql_test 8.3 { finish_test - From fea870be3888ba79a03886205c7a2ba7763177f3 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Aug 2015 20:54:06 +0000 Subject: [PATCH 36/54] Remove some redundant code: Call sqlite3ResolveExprListNames() rather than calling sqlite3ResolveExprNames() in a loop - in two places. FossilOrigin-Name: bdaf66465b6b1bdad10c08d9527b98e7000a41e4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/insert.c | 8 +++++--- src/resolve.c | 9 +-------- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index bc354b2e92..9efff535b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhances\sthe\sparser\sso\sthat\sit\saccepts\sarbitrary\sexpressions\sfor\sthe\sarguments\nof\san\sindex,\sthough\sthe\scode\sgenerator\sstill\srejects\severything\sother\sthan\nsimple\scolumn\snames.\s\sThe\ssqlite3RestrictColumnListSyntax()\sroutine\sis\sremoved\nsince\sthat\sfeature\sis\snow\shandled\sby\sthe\sparser. -D 2015-08-24T20:21:20.985 +C Remove\ssome\sredundant\scode:\s\sCall\ssqlite3ResolveExprListNames()\srather\sthan\ncalling\ssqlite3ResolveExprNames()\sin\sa\sloop\s-\sin\stwo\splaces. +D 2015-08-24T20:54:06.120 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -297,7 +297,7 @@ F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 842026863c726df5420f896a8c10eb3e150f9864 +F src/insert.c ad9ebaafdc4438bb0de58cd7d6bc199fb5b6917a F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712 @@ -335,7 +335,7 @@ F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c 7a67cd2aebc9a9eeecd1d104eb6a9237388eb452 +F src/resolve.c 66b2740075fdb8baf90155180d33d9850cbcc976 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c da6d1e7a4f1c8d713ed5415b5ed21d82ef465c0f F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 @@ -1379,7 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d794b34da6f9c77dfe17773b0b17b22de72cce7f -R 551ba7ed9667cd3021493a6e99ba80fb +P bed42116addabcf3dfdc2e2d51ae183965704988 +R 455d327d4502b1aadc3594ff31ef63f2 U drh -Z 12005a1b68dc81ba8d64d2d3b63a147c +Z 983b791d1ba8013f2148572b484fe05d diff --git a/manifest.uuid b/manifest.uuid index defc8a5329..96012b923f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bed42116addabcf3dfdc2e2d51ae183965704988 \ No newline at end of file +bdaf66465b6b1bdad10c08d9527b98e7000a41e4 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 05d84df844..839599438c 100644 --- a/src/insert.c +++ b/src/insert.c @@ -705,11 +705,13 @@ void sqlite3Insert( sNC.pParse = pParse; srcTab = -1; assert( useTempTable==0 ); - nColumn = pList ? pList->nExpr : 0; - for(i=0; ia[i].pExpr) ){ + if( pList ){ + nColumn = pList->nExpr; + if( sqlite3ResolveExprListNames(&sNC, pList) ){ goto insert_cleanup; } + }else{ + nColumn = 0; } } diff --git a/src/resolve.c b/src/resolve.c index c859e886a7..4ef8fe051b 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1507,7 +1507,6 @@ void sqlite3ResolveSelfReference( ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ - int i; /* Loop counter */ assert( type==NC_IsCheck || type==NC_PartIdx ); memset(&sNC, 0, sizeof(sNC)); @@ -1520,11 +1519,5 @@ void sqlite3ResolveSelfReference( sNC.pSrcList = &sSrc; sNC.ncFlags = type; if( sqlite3ResolveExprNames(&sNC, pExpr) ) return; - if( pList ){ - for(i=0; inExpr; i++){ - if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ - return; - } - } - } + if( pList ) sqlite3ResolveExprListNames(&sNC, pList); } From f8febc1b052ae908f1f95789bc1a630e9fd5666d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Aug 2015 00:34:54 +0000 Subject: [PATCH 37/54] Fix a memory leak that might occur when compiling with SQLITE_OMIT_CHECK. FossilOrigin-Name: 8f1d9f1f308518522ebb1eaebb9b184b4ac30924 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/build.c | 2 -- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9efff535b2..dc6880d48e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sredundant\scode:\s\sCall\ssqlite3ResolveExprListNames()\srather\sthan\ncalling\ssqlite3ResolveExprNames()\sin\sa\sloop\s-\sin\stwo\splaces. -D 2015-08-24T20:54:06.120 +C Fix\sa\smemory\sleak\sthat\smight\soccur\swhen\scompiling\swith\sSQLITE_OMIT_CHECK. +D 2015-08-25T00:34:54.628 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c 789e75f3478ac63c0f398a131c49a0802c356c2b +F src/build.c 6debb6244f38bdcadaed1042a71bdb8e6a471daf F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -1379,7 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bed42116addabcf3dfdc2e2d51ae183965704988 -R 455d327d4502b1aadc3594ff31ef63f2 +P bdaf66465b6b1bdad10c08d9527b98e7000a41e4 +R c205d3ef222cb4a1d4248bf7f5251545 U drh -Z 983b791d1ba8013f2148572b484fe05d +Z 581df61ac20e484fcd60729c96ccadec diff --git a/manifest.uuid b/manifest.uuid index 96012b923f..ebcb1d830a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bdaf66465b6b1bdad10c08d9527b98e7000a41e4 \ No newline at end of file +8f1d9f1f308518522ebb1eaebb9b184b4ac30924 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 8a7dda89c1..ae602378f5 100644 --- a/src/build.c +++ b/src/build.c @@ -640,9 +640,7 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); -#ifndef SQLITE_OMIT_CHECK sqlite3ExprListDelete(db, pTable->pCheck); -#endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); #endif From e91076981782e96e4c9acb79cff227f19a6425ae Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Aug 2015 19:20:04 +0000 Subject: [PATCH 38/54] Use the sqlite3IndexColumnAffinity() routine to quickly and correctly find the affinity of an index column. FossilOrigin-Name: 1ee089a72d789002a0a377347fc51e08ab32fb14 --- manifest | 26 +++++++++++++------------- manifest.uuid | 2 +- src/delete.c | 2 +- src/fkey.c | 2 +- src/insert.c | 13 +++++++++++-- src/pragma.c | 2 +- src/sqliteInt.h | 3 ++- src/update.c | 2 +- src/where.c | 11 ++++------- src/wherecode.c | 2 +- 10 files changed, 36 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index dc6880d48e..98a1fd3f66 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sthat\smight\soccur\swhen\scompiling\swith\sSQLITE_OMIT_CHECK. -D 2015-08-25T00:34:54.628 +C Use\sthe\ssqlite3IndexColumnAffinity()\sroutine\sto\squickly\sand\scorrectly\sfind\sthe\naffinity\sof\san\sindex\scolumn. +D 2015-08-25T19:20:04.670 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,16 +288,16 @@ F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c 8ec787fed4929d8ccdf6b1bc360fccc3e1d2ca58 F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a -F src/delete.c 8857a6f27560718f65d43bdbec86c967ae1f8dfa +F src/delete.c 813be7b5659d7658c8a71b5ae194b45c8f739c8b F src/expr.c 650ac7c4f659980a3315e2aaa02a0d71e87f14a5 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c c9b63a217d86582c22121699a47f22f524608869 +F src/fkey.c 3ce33dd49f12c72376cec9adc7a4d8e7111cedcc F src/func.c 824bea430d3a2b7dbc62806ad54da8fdb8ed9e3f F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c ad9ebaafdc4438bb0de58cd7d6bc199fb5b6917a +F src/insert.c 0edf54c489f5752194b9263cb0a78c247cc4e580 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712 @@ -330,7 +330,7 @@ F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c a3fe31b17e841ec70beee72a2c960e9c787a8857 -F src/pragma.c 669bc0fdb3fb5554e18330e8dd9743319bac16f4 +F src/pragma.c a239d2c8c6d87d589927547f234b0f6259c69f62 F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc @@ -342,7 +342,7 @@ F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h edbcd0c0787541a636a25ab1d1eaf847dbd043f1 +F src/sqliteInt.h fdf60b248e260a4189e6d37a3dfd3500b6bbe629 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -396,7 +396,7 @@ F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 F src/tokenize.c 57cb3720f53f84d811def2069c2b169b6be539a5 F src/treeview.c c15df00728034549ff92d78ae851b44952736d3b F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f -F src/update.c 487747b328b7216bb7f6af0695d6937d5c9e605f +F src/update.c adc8b4b2b6fd2cca2e0d2b803e0cf6956aa3a030 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c bc9dd64b5db544218b871b66243871c202b2781f F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 @@ -414,9 +414,9 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c 66518a14a1238611aa0744d6980b6b7f544f4816 +F src/where.c 30091fb355971e86fc4b6fa709ff7edda4b7a7d8 F src/whereInt.h 880a8599226ac1c00203490d934f3ed79b292572 -F src/wherecode.c 69f19535a6de0cceb10e16b31a3a03463e31bc24 +F src/wherecode.c 3d9113cc307ffeed58db41fe9f2d807c94787ab5 F src/whereexpr.c 1a308d1ee5144890d21ea9cf70d49bc96a83432b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1379,7 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bdaf66465b6b1bdad10c08d9527b98e7000a41e4 -R c205d3ef222cb4a1d4248bf7f5251545 +P 8f1d9f1f308518522ebb1eaebb9b184b4ac30924 +R 9f42504abba31a08ea4c89fa9c98aeeb U drh -Z 581df61ac20e484fcd60729c96ccadec +Z 95c550a2c6ae0fd191dda0e3160d77d8 diff --git a/manifest.uuid b/manifest.uuid index ebcb1d830a..757ef27e96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f1d9f1f308518522ebb1eaebb9b184b4ac30924 \ No newline at end of file +1ee089a72d789002a0a377347fc51e08ab32fb14 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 369cdaf6fe..917157743c 100644 --- a/src/delete.c +++ b/src/delete.c @@ -443,7 +443,7 @@ void sqlite3DeleteFrom( iKey = ++pParse->nMem; nKey = 0; /* Zero tells OP_Found to use a composite key */ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, - sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3IndexAffinityStr(pParse->db, pPk), nPk); sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); }else{ /* Get the rowid of the row to be deleted and remember it in the RowSet */ diff --git a/src/fkey.c b/src/fkey.c index 09513e4620..9d448def6b 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -416,7 +416,7 @@ static void fkLookupParent( } sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, - sqlite3IndexAffinityStr(v,pIdx), nCol); + sqlite3IndexAffinityStr(pParse->db,pIdx), nCol); sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, regRec); diff --git a/src/insert.c b/src/insert.c index 839599438c..e94af09855 100644 --- a/src/insert.c +++ b/src/insert.c @@ -69,7 +69,7 @@ void sqlite3OpenTable( ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ +const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ if( !pIdx->zColAff ){ /* The first time a column affinity string for a particular index is ** required, it is allocated and populated here. It is then stored as @@ -81,7 +81,6 @@ const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ */ int n; Table *pTab = pIdx->pTable; - sqlite3 *db = sqlite3VdbeDb(v); pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ db->mallocFailed = 1; @@ -97,6 +96,16 @@ const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ return pIdx->zColAff; } +/* +** Return the affinity for a single column of an index. +*/ +char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ + if( !pIdx->zColAff ){ + if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; + } + return pIdx->zColAff[iCol]; +} + /* ** Compute the affinity string for table pTab, if it has not already been ** computed. As an optimization, omit trailing SQLITE_AFF_BLOB affinities. diff --git a/src/pragma.c b/src/pragma.c index 96ff136c25..96a0272dea 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1342,7 +1342,7 @@ void sqlite3Pragma( } if( pParent ){ sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, - sqlite3IndexAffinityStr(v,pIdx), pFK->nCol); + sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); VdbeCoverage(v); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4c17904ff8..288b256168 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3550,7 +3550,8 @@ int sqlite3VarintLen(u64 v); #define putVarint sqlite3PutVarint -const char *sqlite3IndexAffinityStr(Vdbe *, Index *); +const char *sqlite3IndexAffinityStr(sqlite3*, Index*); +char sqlite3IndexColumnAffinity(sqlite3*, Index*, int); void sqlite3TableAffinity(Vdbe*, Table*, int); char sqlite3CompareAffinity(Expr *pExpr, char aff2); int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); diff --git a/src/update.c b/src/update.c index f8347448a1..a8bcd4efa0 100644 --- a/src/update.c +++ b/src/update.c @@ -390,7 +390,7 @@ void sqlite3Update( regKey = iPk; }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, - sqlite3IndexAffinityStr(v, pPk), nPk); + sqlite3IndexAffinityStr(db, pPk), nPk); sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey); } sqlite3WhereEnd(pWInfo); diff --git a/src/where.c b/src/where.c index bdcfeaa9aa..deebed83bd 100644 --- a/src/where.c +++ b/src/where.c @@ -1177,7 +1177,7 @@ static int whereRangeSkipScanEst( int nUpper = p->nSample+1; int rc = SQLITE_OK; int iCol = p->aiColumn[nEq]; - u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; + u8 aff = sqlite3IndexColumnAffinity(db, p, iCol); CollSeq *pColl; sqlite3_value *p1 = 0; /* Value extracted from pLower */ @@ -1325,11 +1325,8 @@ static int whereRangeScanEst( testcase( pRec->nField!=pBuilder->nRecValid ); pRec->nField = pBuilder->nRecValid; } - if( nEq==p->nKeyCol ){ - aff = SQLITE_AFF_INTEGER; - }else{ - aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; - } + aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq); + assert( nEq!=p->nKeyCol || aff==SQLITE_AFF_INTEGER ); /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; @@ -1487,7 +1484,7 @@ static int whereEqualScanEst( return SQLITE_OK; } - aff = p->pTable->aCol[p->aiColumn[nEq-1]].affinity; + aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq-1); rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); pBuilder->pRec = pRec; if( rc!=SQLITE_OK ) return rc; diff --git a/src/wherecode.c b/src/wherecode.c index eb23d8f1a2..e5c0b40b1a 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -492,7 +492,7 @@ static int codeAllEqualityTerms( nReg = pLoop->u.btree.nEq + nExtraReg; pParse->nMem += nReg; - zAff = sqlite3DbStrDup(pParse->db, sqlite3IndexAffinityStr(v, pIdx)); + zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); if( !zAff ){ pParse->db->mallocFailed = 1; } From 567cc1e407921fe87f808cc856ced9896eb83a2a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Aug 2015 19:42:28 +0000 Subject: [PATCH 39/54] Move sqlite3IndexColumnAffinity() inside of SQLITE_ENABLE_STAT3_OR_STAT4. FossilOrigin-Name: b3732a4e1b42a86675d89766cb8089914e2fab9a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/insert.c | 10 ---------- src/sqliteInt.h | 1 - src/where.c | 14 ++++++++++++++ 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 98a1fd3f66..c06939334e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sthe\ssqlite3IndexColumnAffinity()\sroutine\sto\squickly\sand\scorrectly\sfind\sthe\naffinity\sof\san\sindex\scolumn. -D 2015-08-25T19:20:04.670 +C Move\ssqlite3IndexColumnAffinity()\sinside\sof\sSQLITE_ENABLE_STAT3_OR_STAT4. +D 2015-08-25T19:42:28.613 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -297,7 +297,7 @@ F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 0edf54c489f5752194b9263cb0a78c247cc4e580 +F src/insert.c c31b9253f0d40425d012701ba6e7e4b28c1676ea F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712 @@ -342,7 +342,7 @@ F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h fdf60b248e260a4189e6d37a3dfd3500b6bbe629 +F src/sqliteInt.h 424a2020efc9736c47667db06a95e5c580782798 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -414,7 +414,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c 30091fb355971e86fc4b6fa709ff7edda4b7a7d8 +F src/where.c fb8546b8053433bfb39f6c39fdc99c111a6f97c5 F src/whereInt.h 880a8599226ac1c00203490d934f3ed79b292572 F src/wherecode.c 3d9113cc307ffeed58db41fe9f2d807c94787ab5 F src/whereexpr.c 1a308d1ee5144890d21ea9cf70d49bc96a83432b @@ -1379,7 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8f1d9f1f308518522ebb1eaebb9b184b4ac30924 -R 9f42504abba31a08ea4c89fa9c98aeeb +P 1ee089a72d789002a0a377347fc51e08ab32fb14 +R ae8d80bca7ce576313f5297ac0cf95e0 U drh -Z 95c550a2c6ae0fd191dda0e3160d77d8 +Z 2b2ee5ff170c2b474ba393d9405c67b9 diff --git a/manifest.uuid b/manifest.uuid index 757ef27e96..d0b8abd542 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ee089a72d789002a0a377347fc51e08ab32fb14 \ No newline at end of file +b3732a4e1b42a86675d89766cb8089914e2fab9a \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index e94af09855..c6a0ad7057 100644 --- a/src/insert.c +++ b/src/insert.c @@ -96,16 +96,6 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ return pIdx->zColAff; } -/* -** Return the affinity for a single column of an index. -*/ -char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ - if( !pIdx->zColAff ){ - if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; - } - return pIdx->zColAff[iCol]; -} - /* ** Compute the affinity string for table pTab, if it has not already been ** computed. As an optimization, omit trailing SQLITE_AFF_BLOB affinities. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 288b256168..a57f00afb8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3551,7 +3551,6 @@ int sqlite3VarintLen(u64 v); const char *sqlite3IndexAffinityStr(sqlite3*, Index*); -char sqlite3IndexColumnAffinity(sqlite3*, Index*, int); void sqlite3TableAffinity(Vdbe*, Table*, int); char sqlite3CompareAffinity(Expr *pExpr, char aff2); int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); diff --git a/src/where.c b/src/where.c index deebed83bd..9789b0848d 100644 --- a/src/where.c +++ b/src/where.c @@ -1127,6 +1127,20 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ return nRet; } + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** Return the affinity for a single column of an index. +*/ +static char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ + if( !pIdx->zColAff ){ + if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; + } + return pIdx->zColAff[iCol]; +} +#endif + + #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** This function is called to estimate the number of rows visited by a From 1c4505de91f059bcfab8b59395455010581d43fb Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Aug 2015 11:34:31 +0000 Subject: [PATCH 40/54] Update the TreeView output for compound SELECT statements so that all entries are shown vertically aligned rather than each successive entry being indented. FossilOrigin-Name: 65a8918776aa395009a690fa86bfc7d99eb973f9 --- manifest | 12 ++-- manifest.uuid | 2 +- src/treeview.c | 171 +++++++++++++++++++++++++------------------------ 3 files changed, 96 insertions(+), 89 deletions(-) diff --git a/manifest b/manifest index c06939334e..999ede22cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\ssqlite3IndexColumnAffinity()\sinside\sof\sSQLITE_ENABLE_STAT3_OR_STAT4. -D 2015-08-25T19:42:28.613 +C Update\sthe\sTreeView\soutput\sfor\scompound\sSELECT\sstatements\sso\sthat\sall\sentries\nare\sshown\svertically\saligned\srather\sthan\seach\ssuccessive\sentry\sbeing\sindented. +D 2015-08-26T11:34:31.015 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -394,7 +394,7 @@ F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 F src/tokenize.c 57cb3720f53f84d811def2069c2b169b6be539a5 -F src/treeview.c c15df00728034549ff92d78ae851b44952736d3b +F src/treeview.c 24950c6a79583016c83c43c8b741b3b79a096ce8 F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/update.c adc8b4b2b6fd2cca2e0d2b803e0cf6956aa3a030 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c @@ -1379,7 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1ee089a72d789002a0a377347fc51e08ab32fb14 -R ae8d80bca7ce576313f5297ac0cf95e0 +P b3732a4e1b42a86675d89766cb8089914e2fab9a +R df971f9d2c0125dc9d07b8638209a38a U drh -Z 2b2ee5ff170c2b474ba393d9405c67b9 +Z 2faf27f44155dca5bb6f2deaf5ffdfce diff --git a/manifest.uuid b/manifest.uuid index d0b8abd542..67a7b1df78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b3732a4e1b42a86675d89766cb8089914e2fab9a \ No newline at end of file +65a8918776aa395009a690fa86bfc7d99eb973f9 \ No newline at end of file diff --git a/src/treeview.c b/src/treeview.c index fbe8fd46d5..57538643e0 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -85,93 +85,100 @@ static void sqlite3TreeViewItem(TreeView *p, const char *zLabel,u8 moreFollows){ */ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ int n = 0; + int cnt = 0; pView = sqlite3TreeViewPush(pView, moreToFollow); - sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x", - ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags - ); - if( p->pSrc && p->pSrc->nSrc ) n++; - if( p->pWhere ) n++; - if( p->pGroupBy ) n++; - if( p->pHaving ) n++; - if( p->pOrderBy ) n++; - if( p->pLimit ) n++; - if( p->pOffset ) n++; - if( p->pPrior ) n++; - sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); - if( p->pSrc && p->pSrc->nSrc ){ - int i; - pView = sqlite3TreeViewPush(pView, (n--)>0); - sqlite3TreeViewLine(pView, "FROM"); - for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - StrAccum x; - char zLine[100]; - sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); - sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); - if( pItem->zDatabase ){ - sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); - }else if( pItem->zName ){ - sqlite3XPrintf(&x, 0, " %s", pItem->zName); - } - if( pItem->pTab ){ - sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); - } - if( pItem->zAlias ){ - sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); - } - if( pItem->fg.jointype & JT_LEFT ){ - sqlite3XPrintf(&x, 0, " LEFT-JOIN"); - } - sqlite3StrAccumFinish(&x); - sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); - if( pItem->pSelect ){ - sqlite3TreeViewSelect(pView, pItem->pSelect, 0); - } - if( pItem->fg.isTabFunc ){ - sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + do{ + sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags + ); + if( cnt++ ) sqlite3TreeViewPop(pView); + if( p->pPrior ){ + n = 1000; + }else{ + n = 0; + if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pWhere ) n++; + if( p->pGroupBy ) n++; + if( p->pHaving ) n++; + if( p->pOrderBy ) n++; + if( p->pLimit ) n++; + if( p->pOffset ) n++; + } + sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); + if( p->pSrc && p->pSrc->nSrc ){ + int i; + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "FROM"); + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); + }else if( pItem->zName ){ + sqlite3XPrintf(&x, 0, " %s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); + } + if( pItem->zAlias ){ + sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); + } + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3XPrintf(&x, 0, " LEFT-JOIN"); + } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + if( pItem->fg.isTabFunc ){ + sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + } + sqlite3TreeViewPop(pView); } sqlite3TreeViewPop(pView); } - sqlite3TreeViewPop(pView); - } - if( p->pWhere ){ - sqlite3TreeViewItem(pView, "WHERE", (n--)>0); - sqlite3TreeViewExpr(pView, p->pWhere, 0); - sqlite3TreeViewPop(pView); - } - if( p->pGroupBy ){ - sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); - } - if( p->pHaving ){ - sqlite3TreeViewItem(pView, "HAVING", (n--)>0); - sqlite3TreeViewExpr(pView, p->pHaving, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOrderBy ){ - sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); - } - if( p->pLimit ){ - sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); - sqlite3TreeViewExpr(pView, p->pLimit, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOffset ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); - sqlite3TreeViewExpr(pView, p->pOffset, 0); - sqlite3TreeViewPop(pView); - } - if( p->pPrior ){ - const char *zOp = "UNION"; - switch( p->op ){ - case TK_ALL: zOp = "UNION ALL"; break; - case TK_INTERSECT: zOp = "INTERSECT"; break; - case TK_EXCEPT: zOp = "EXCEPT"; break; + if( p->pWhere ){ + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, p->pWhere, 0); + sqlite3TreeViewPop(pView); } - sqlite3TreeViewItem(pView, zOp, (n--)>0); - sqlite3TreeViewSelect(pView, p->pPrior, 0); - sqlite3TreeViewPop(pView); - } + if( p->pGroupBy ){ + sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); + } + if( p->pHaving ){ + sqlite3TreeViewItem(pView, "HAVING", (n--)>0); + sqlite3TreeViewExpr(pView, p->pHaving, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOrderBy ){ + sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); + } + if( p->pLimit ){ + sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit, 0); + sqlite3TreeViewPop(pView); + } + if( p->pOffset ){ + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewExpr(pView, p->pOffset, 0); + sqlite3TreeViewPop(pView); + } + if( p->pPrior ){ + const char *zOp = "UNION"; + switch( p->op ){ + case TK_ALL: zOp = "UNION ALL"; break; + case TK_INTERSECT: zOp = "INTERSECT"; break; + case TK_EXCEPT: zOp = "EXCEPT"; break; + } + sqlite3TreeViewItem(pView, zOp, 1); + } + p = p->pPrior; + }while( p!=0 ); sqlite3TreeViewPop(pView); } From 0576bc59a730c57f14f037dc90a28d0a7ec2d51f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Aug 2015 11:40:11 +0000 Subject: [PATCH 41/54] Refactor With.a.zErr into With.a.zCteErr. No logic changes. FossilOrigin-Name: 58ba73630ecc4bc58b03a7962dd45b305ef605ef --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/build.c | 2 +- src/select.c | 16 ++++++++-------- src/sqliteInt.h | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 999ede22cf..6f175329a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sTreeView\soutput\sfor\scompound\sSELECT\sstatements\sso\sthat\sall\sentries\nare\sshown\svertically\saligned\srather\sthan\seach\ssuccessive\sentry\sbeing\sindented. -D 2015-08-26T11:34:31.015 +C Refactor\sWith.a.zErr\sinto\sWith.a.zCteErr.\s\sNo\slogic\schanges. +D 2015-08-26T11:40:11.832 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c 6debb6244f38bdcadaed1042a71bdb8e6a471daf +F src/build.c 97f682229876834abad515a0a48759a967999ed0 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -337,12 +337,12 @@ F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 66b2740075fdb8baf90155180d33d9850cbcc976 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c da6d1e7a4f1c8d713ed5415b5ed21d82ef465c0f +F src/select.c 50b0f02ec4482f92749e6630de9cd0d175a040e1 F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h 424a2020efc9736c47667db06a95e5c580782798 +F src/sqliteInt.h d76e7c90775efeec72ea254a5da0a9f1ddcff765 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -1379,7 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b3732a4e1b42a86675d89766cb8089914e2fab9a -R df971f9d2c0125dc9d07b8638209a38a +P 65a8918776aa395009a690fa86bfc7d99eb973f9 +R a6b3114ece5e49154c10e2c8118e9866 U drh -Z 2faf27f44155dca5bb6f2deaf5ffdfce +Z b1a98f3cfbd2154f3cffafb179efca91 diff --git a/manifest.uuid b/manifest.uuid index 67a7b1df78..867315d21b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65a8918776aa395009a690fa86bfc7d99eb973f9 \ No newline at end of file +58ba73630ecc4bc58b03a7962dd45b305ef605ef \ No newline at end of file diff --git a/src/build.c b/src/build.c index ae602378f5..0960d0cbdf 100644 --- a/src/build.c +++ b/src/build.c @@ -4349,7 +4349,7 @@ With *sqlite3WithAdd( pNew->a[pNew->nCte].pSelect = pQuery; pNew->a[pNew->nCte].pCols = pArglist; pNew->a[pNew->nCte].zName = zName; - pNew->a[pNew->nCte].zErr = 0; + pNew->a[pNew->nCte].zCteErr = 0; pNew->nCte++; } diff --git a/src/select.c b/src/select.c index 2581a240e9..dbf06dd9ad 100644 --- a/src/select.c +++ b/src/select.c @@ -4053,12 +4053,12 @@ static int withExpand( int bMayRecursive; /* True if compound joined by UNION [ALL] */ With *pSavedWith; /* Initial value of pParse->pWith */ - /* If pCte->zErr is non-NULL at this point, then this is an illegal + /* If pCte->zCteErr is non-NULL at this point, then this is an illegal ** recursive reference to CTE pCte. Leave an error in pParse and return - ** early. If pCte->zErr is NULL, then this is not a recursive reference. + ** early. If pCte->zCteErr is NULL, then this is not a recursive reference. ** In this case, proceed. */ - if( pCte->zErr ){ - sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); + if( pCte->zCteErr ){ + sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName); return SQLITE_ERROR; } @@ -4103,7 +4103,7 @@ static int withExpand( } assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); - pCte->zErr = "circular reference: %s"; + pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); @@ -4124,13 +4124,13 @@ static int withExpand( sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); if( bMayRecursive ){ if( pSel->selFlags & SF_Recursive ){ - pCte->zErr = "multiple recursive references: %s"; + pCte->zCteErr = "multiple recursive references: %s"; }else{ - pCte->zErr = "recursive reference in a subquery: %s"; + pCte->zCteErr = "recursive reference in a subquery: %s"; } sqlite3WalkSelect(pWalker, pSel); } - pCte->zErr = 0; + pCte->zCteErr = 0; pParse->pWith = pSavedWith; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a57f00afb8..867a608ec0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3030,7 +3030,7 @@ struct With { char *zName; /* Name of this CTE */ ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The definition of this CTE */ - const char *zErr; /* Error message for circular references */ + const char *zCteErr; /* Error message for circular references */ } a[1]; }; From 5579d59fb3ccb9e9db9913a5bd50871d8e893d95 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Aug 2015 14:01:41 +0000 Subject: [PATCH 42/54] Evaluate expressions only once when the same expression is used in both the result set and in the ORDER BY clause. FossilOrigin-Name: c2f3bbad778504681b39ab9399a1eb3c1a35ab3f --- manifest | 21 ++++++++++--------- manifest.uuid | 2 +- src/expr.c | 11 ++++++---- src/resolve.c | 4 ++-- src/select.c | 20 +++++++++++------- src/sqliteInt.h | 3 ++- src/treeview.c | 6 ++++++ test/orderby9.test | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 93 insertions(+), 26 deletions(-) create mode 100644 test/orderby9.test diff --git a/manifest b/manifest index 6f175329a1..5a48ac0a18 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sWith.a.zErr\sinto\sWith.a.zCteErr.\s\sNo\slogic\schanges. -D 2015-08-26T11:40:11.832 +C Evaluate\sexpressions\sonly\sonce\swhen\sthe\ssame\sexpression\sis\sused\sin\sboth\sthe\nresult\sset\sand\sin\sthe\sORDER\sBY\sclause. +D 2015-08-26T14:01:41.658 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c 8ec787fed4929d8ccdf6b1bc360fccc3e1d2ca58 F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a F src/delete.c 813be7b5659d7658c8a71b5ae194b45c8f739c8b -F src/expr.c 650ac7c4f659980a3315e2aaa02a0d71e87f14a5 +F src/expr.c 5944e529891416f482250e16c598d8c26e149eb0 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 3ce33dd49f12c72376cec9adc7a4d8e7111cedcc F src/func.c 824bea430d3a2b7dbc62806ad54da8fdb8ed9e3f @@ -335,14 +335,14 @@ F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c 66b2740075fdb8baf90155180d33d9850cbcc976 +F src/resolve.c f2ef256786a6435efddd64a632fea89c8be62215 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 50b0f02ec4482f92749e6630de9cd0d175a040e1 +F src/select.c b52c80f2b1bdb62491f9ce40eea0c5f80c78d105 F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h d76e7c90775efeec72ea254a5da0a9f1ddcff765 +F src/sqliteInt.h cac6c31a0c7e6aa5572cc97b68d2630034d6d212 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -394,7 +394,7 @@ F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 F src/tokenize.c 57cb3720f53f84d811def2069c2b169b6be539a5 -F src/treeview.c 24950c6a79583016c83c43c8b741b3b79a096ce8 +F src/treeview.c 46036cbbceada0836833531b2d963edbca3d9cfa F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/update.c adc8b4b2b6fd2cca2e0d2b803e0cf6956aa3a030 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c @@ -899,6 +899,7 @@ F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd +F test/orderby9.test 88a330ea5fc7bed7e407b28beb0d2b79485ae2cc F test/oserror.test 14fec2796c2b6fe431c7823750e8a18a761176d7 F test/ovfl.test 4f7ca651cba5c059a12d8c67dddd49bec5747799 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa @@ -1379,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 65a8918776aa395009a690fa86bfc7d99eb973f9 -R a6b3114ece5e49154c10e2c8118e9866 +P 58ba73630ecc4bc58b03a7962dd45b305ef605ef +R 657299463d01da646731fb6b8ab7d9fe U drh -Z b1a98f3cfbd2154f3cffafb179efca91 +Z 83e3936f8767c9500521d2082aa870a1 diff --git a/manifest.uuid b/manifest.uuid index 867315d21b..1045acaa92 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58ba73630ecc4bc58b03a7962dd45b305ef605ef \ No newline at end of file +c2f3bbad778504681b39ab9399a1eb3c1a35ab3f \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1aebef6b16..71c552c687 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2912,7 +2912,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ } sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ - sqlite3ExprCodeExprList(pParse, pFarg, r1, + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ }else{ @@ -3328,11 +3328,13 @@ int sqlite3ExprCodeExprList( Parse *pParse, /* Parsing context */ ExprList *pList, /* The expression list to be coded */ int target, /* Where to write results */ + int srcReg, /* Source registers if SQLITE_ECEL_REF */ u8 flags /* SQLITE_ECEL_* flags */ ){ struct ExprList_item *pItem; - int i, n; + int i, j, n; u8 copyOp = (flags & SQLITE_ECEL_DUP) ? OP_Copy : OP_SCopy; + Vdbe *v = pParse->pVdbe; assert( pList!=0 ); assert( target>0 ); assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ @@ -3340,13 +3342,14 @@ int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; - if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ + if( (flags & SQLITE_ECEL_REF)!=0 && (j = pList->a[i].u.x.iOrderByCol)>0 ){ + sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); }else{ int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); if( inReg!=target+i ){ VdbeOp *pOp; - Vdbe *v = pParse->pVdbe; if( copyOp==OP_Copy && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg diff --git a/src/resolve.c b/src/resolve.c index 4ef8fe051b..04fa8429a8 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -407,9 +407,9 @@ static int lookupName( ** resolved by the time the WHERE clause is resolved. ** ** The ability to use an output result-set column in the WHERE, GROUP BY, - ** or HAVING clauses, or as part of a larger expression in the ORDRE BY + ** or HAVING clauses, or as part of a larger expression in the ORDER BY ** clause is not standard SQL. This is a (goofy) SQLite extension, that - ** is supported for backwards compatibility only. TO DO: Issue a warning + ** is supported for backwards compatibility only. Hence, we issue a warning ** on sqlite3_log() whenever the capability is used. */ if( (pEList = pNC->pEList)!=0 diff --git a/src/select.c b/src/select.c index dbf06dd9ad..21366b9d11 100644 --- a/src/select.c +++ b/src/select.c @@ -496,6 +496,7 @@ static void pushOntoSorter( SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData, /* First register holding data to be sorted */ + int regOrigData, /* First register holding data before packing */ int nData, /* Number of elements in the data array */ int nPrefixReg /* No. of reg prior to regData available for use */ ){ @@ -509,6 +510,7 @@ static void pushOntoSorter( int op; /* Opcode to add sorter record to sorter */ assert( bSeq==0 || bSeq==1 ); + assert( nData==1 || regData==regOrigData ); if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nExpr - bSeq; @@ -516,7 +518,8 @@ static void pushOntoSorter( regBase = pParse->nMem + 1; pParse->nMem += nBase; } - sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, + SQLITE_ECEL_DUP|SQLITE_ECEL_REF); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } @@ -726,7 +729,7 @@ static void selectInnerLoop( }else{ ecelFlags = 0; } - sqlite3ExprCodeExprList(pParse, pEList, regResult, ecelFlags); + sqlite3ExprCodeExprList(pParse, pEList, regResult, 0, ecelFlags); } /* If the DISTINCT keyword was present on the SELECT statement @@ -842,7 +845,7 @@ static void selectInnerLoop( } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg,regResult,1,nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -868,7 +871,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -894,7 +897,7 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); }else{ assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ @@ -908,7 +911,8 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg); + pushOntoSorter(pParse, pSort, p, regResult, regResult, nResultCol, + nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ @@ -4667,7 +4671,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, SQLITE_ECEL_DUP); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ nArg = 0; regAgg = 0; @@ -5287,7 +5291,7 @@ int sqlite3Select( } regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; for(i=0; inExpr; i++){ + int j = pList->a[i].u.x.iOrderByCol; + if( j ){ + sqlite3TreeViewPush(pView, 0); + sqlite3TreeViewLine(pView, "iOrderByCol=%d", j); + } sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); + if( j ) sqlite3TreeViewPop(pView); } } sqlite3TreeViewPop(pView); diff --git a/test/orderby9.test b/test/orderby9.test new file mode 100644 index 0000000000..c998c5054e --- /dev/null +++ b/test/orderby9.test @@ -0,0 +1,52 @@ +# 2015-08-26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file seeks to verify that expressions (and especially functions) +# that are in both the ORDER BY clause and the result set are only +# evaluated once. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderby9 + + +do_execsql_test setup { + -- create a table with many entries + CREATE TABLE t1(x); + WITH RECURSIVE + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1 SELECT x FROM c; +} +do_test 1.0 { + set l1 {} + # If random() is only evaluated once and then reused for each row, then + # the output should appear in sorted order. If random() is evaluated + # separately for the result set and the ORDER BY clause, then the output + # order will be random. + db eval {SELECT random() AS y FROM t1 ORDER BY 1;} {lappend l1 $y} + expr {$l1==[lsort -int $l1]} +} {1} + +do_test 1.1 { + set l1 {} + db eval {SELECT random() AS y FROM t1 ORDER BY random();} {lappend l1 $y} + expr {$l1==[lsort -int $l1]} +} {1} + +do_test 1.2 { + set l1 {} + db eval {SELECT random() AS y FROM t1 ORDER BY +random();} {lappend l1 $y} + expr {$l1==[lsort -int $l1]} +} {0} + +finish_test From a3f108e93dfa0576604c10a8590750ca611e840d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Aug 2015 21:08:04 +0000 Subject: [PATCH 43/54] Reduce the size of the WhereScan object by 24 bytes while also clarifying its operation. FossilOrigin-Name: cbc3c9a8bf169ae0b21f26855038502c6cc25cfe --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 46 ++++++++++++++++++++++------------------------ src/whereInt.h | 3 ++- 4 files changed, 32 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 5a48ac0a18..13b7cb4d99 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Evaluate\sexpressions\sonly\sonce\swhen\sthe\ssame\sexpression\sis\sused\sin\sboth\sthe\nresult\sset\sand\sin\sthe\sORDER\sBY\sclause. -D 2015-08-26T14:01:41.658 +C Reduce\sthe\ssize\sof\sthe\sWhereScan\sobject\sby\s24\sbytes\swhile\salso\sclarifying\sits\noperation. +D 2015-08-26T21:08:04.505 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -414,8 +414,8 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c fb8546b8053433bfb39f6c39fdc99c111a6f97c5 -F src/whereInt.h 880a8599226ac1c00203490d934f3ed79b292572 +F src/where.c caabc9ec1e86168d7717833700cfed01a75f687a +F src/whereInt.h 901c17c1e3c82745ad9b85b4471543fa59c980e9 F src/wherecode.c 3d9113cc307ffeed58db41fe9f2d807c94787ab5 F src/whereexpr.c 1a308d1ee5144890d21ea9cf70d49bc96a83432b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 58ba73630ecc4bc58b03a7962dd45b305ef605ef -R 657299463d01da646731fb6b8ab7d9fe +P c2f3bbad778504681b39ab9399a1eb3c1a35ab3f +R f5f6b9b31662d17ccccce315f9a81df1 U drh -Z 83e3936f8767c9500521d2082aa870a1 +Z be2744e78cce20e5ac9fa855b3738fc2 diff --git a/manifest.uuid b/manifest.uuid index 1045acaa92..37ba53dcda 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c2f3bbad778504681b39ab9399a1eb3c1a35ab3f \ No newline at end of file +cbc3c9a8bf169ae0b21f26855038502c6cc25cfe \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9789b0848d..3a64ef0af2 100644 --- a/src/where.c +++ b/src/where.c @@ -171,37 +171,37 @@ static void createMask(WhereMaskSet *pMaskSet, int iCursor){ */ static WhereTerm *whereScanNext(WhereScan *pScan){ int iCur; /* The cursor on the LHS of the term */ - int iColumn; /* The column on the LHS of the term. -1 for IPK */ + i16 iColumn; /* The column on the LHS of the term. -1 for IPK */ Expr *pX; /* An expression being tested */ WhereClause *pWC; /* Shorthand for pScan->pWC */ WhereTerm *pTerm; /* The term being tested */ int k = pScan->k; /* Where to start scanning */ while( pScan->iEquiv<=pScan->nEquiv ){ - iCur = pScan->aEquiv[pScan->iEquiv-2]; - iColumn = pScan->aEquiv[pScan->iEquiv-1]; + iCur = pScan->aiCur[pScan->iEquiv-1]; + iColumn = pScan->aiColumn[pScan->iEquiv-1]; while( (pWC = pScan->pWC)!=0 ){ for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn - && (pScan->iEquiv<=2 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) ){ if( (pTerm->eOperator & WO_EQUIV)!=0 - && pScan->nEquivaEquiv) + && pScan->nEquivaiCur) ){ int j; pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); assert( pX->op==TK_COLUMN ); - for(j=0; jnEquiv; j+=2){ - if( pScan->aEquiv[j]==pX->iTable - && pScan->aEquiv[j+1]==pX->iColumn ){ + for(j=0; jnEquiv; j++){ + if( pScan->aiCur[j]==pX->iTable + && pScan->aiColumn[j]==pX->iColumn ){ break; } } if( j==pScan->nEquiv ){ - pScan->aEquiv[j] = pX->iTable; - pScan->aEquiv[j+1] = pX->iColumn; - pScan->nEquiv += 2; + pScan->aiCur[j] = pX->iTable; + pScan->aiColumn[j] = pX->iColumn; + pScan->nEquiv++; } } if( (pTerm->eOperator & pScan->opMask)!=0 ){ @@ -223,8 +223,8 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ } if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0 && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN - && pX->iTable==pScan->aEquiv[0] - && pX->iColumn==pScan->aEquiv[1] + && pX->iTable==pScan->aiCur[0] + && pX->iColumn==pScan->aiColumn[0] ){ testcase( pTerm->eOperator & WO_IS ); continue; @@ -239,7 +239,7 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ } pScan->pWC = pScan->pOrigWC; k = 0; - pScan->iEquiv += 2; + pScan->iEquiv++; } return 0; } @@ -285,10 +285,10 @@ static WhereTerm *whereScanInit( } pScan->opMask = opMask; pScan->k = 0; - pScan->aEquiv[0] = iCur; - pScan->aEquiv[1] = iColumn; - pScan->nEquiv = 2; - pScan->iEquiv = 2; + pScan->aiCur[0] = iCur; + pScan->aiColumn[0] = iColumn; + pScan->nEquiv = 1; + pScan->iEquiv = 1; return whereScanNext(pScan); } @@ -301,12 +301,10 @@ static WhereTerm *whereScanInit( ** The term returned might by Y= if there is another constraint in ** the WHERE clause that specifies that X=Y. Any such constraints will be ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The -** aEquiv[] array holds X and all its equivalents, with each SQL variable -** taking up two slots in aEquiv[]. The first slot is for the cursor number -** and the second is for the column number. There are 22 slots in aEquiv[] -** so that means we can look for X plus up to 10 other equivalent values. -** Hence a search for X will return if X=A1 and A1=A2 and A2=A3 -** and ... and A9=A10 and A10=. +** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11 +** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10 +** other equivalent values. Hence a search for X will return if X=A1 +** and A1=A2 and A2=A3 and ... and A9=A10 and A10=. ** ** If there are multiple terms in the WHERE clause of the form "X " ** then try for the one with no dependencies on - in other words where diff --git a/src/whereInt.h b/src/whereInt.h index 76b665d971..7138b85b25 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -291,7 +291,8 @@ struct WhereScan { unsigned char iEquiv; /* Next unused slot in aEquiv[] */ u32 opMask; /* Acceptable operators */ int k; /* Resume scanning at this->pWC->a[this->k] */ - int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ + int aiCur[11]; /* Cursors in the equivalence class */ + i16 aiColumn[11]; /* Corresponding column number in the eq-class */ }; /* From 2dc292930d004c340eba968789148916f2b89cd4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Aug 2015 23:18:55 +0000 Subject: [PATCH 44/54] Fix the OR-optimization so that it always ignores subplans that do not use an index. FossilOrigin-Name: 66f92a16866e5825363636b9cc4b8f9b29d9e84d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 16 +++++++++------- test/where2.test | 9 +++++++-- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 13b7cb4d99..4c7e7c0346 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\ssize\sof\sthe\sWhereScan\sobject\sby\s24\sbytes\swhile\salso\sclarifying\sits\noperation. -D 2015-08-26T21:08:04.505 +C Fix\sthe\sOR-optimization\sso\sthat\sit\salways\signores\ssubplans\sthat\sdo\snot\nuse\san\sindex. +D 2015-08-27T23:18:55.309 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -414,7 +414,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c caabc9ec1e86168d7717833700cfed01a75f687a +F src/where.c 91e73ffc699c140a59baa03a6b7b060db02bed81 F src/whereInt.h 901c17c1e3c82745ad9b85b4471543fa59c980e9 F src/wherecode.c 3d9113cc307ffeed58db41fe9f2d807c94787ab5 F src/whereexpr.c 1a308d1ee5144890d21ea9cf70d49bc96a83432b @@ -1287,7 +1287,7 @@ F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 1ff3d9f8da0a6c0dc5ccfd38d9225b2cdb5b6afb -F test/where2.test 23fdb5d8e756554aad4ca7ae03de9dd8367a2c6e +F test/where2.test af78c55589cbc82d793449493adba0dc3d659f23 F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e F test/where4.test 68aa5ad796e33816db2078bc0f6de719c7a0e21f F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c2f3bbad778504681b39ab9399a1eb3c1a35ab3f -R f5f6b9b31662d17ccccce315f9a81df1 +P cbc3c9a8bf169ae0b21f26855038502c6cc25cfe +R 3aef897e420a61a50263b8e461848e83 U drh -Z be2744e78cce20e5ac9fa855b3738fc2 +Z b614bfef572643d9445d8b8a4d103409 diff --git a/manifest.uuid b/manifest.uuid index 37ba53dcda..a3b504350e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cbc3c9a8bf169ae0b21f26855038502c6cc25cfe \ No newline at end of file +66f92a16866e5825363636b9cc4b8f9b29d9e84d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3a64ef0af2..0413db3f02 100644 --- a/src/where.c +++ b/src/where.c @@ -1923,18 +1923,20 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** and prereqs. */ if( pBuilder->pOrSet!=0 ){ + if( pTemplate->nLTerm ){ #if WHERETRACE_ENABLED - u16 n = pBuilder->pOrSet->n; - int x = + u16 n = pBuilder->pOrSet->n; + int x = #endif - whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, pTemplate->nOut); #if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); - whereLoopPrint(pTemplate, pBuilder->pWC); - } + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + whereLoopPrint(pTemplate, pBuilder->pWC); + } #endif + } return SQLITE_OK; } diff --git a/test/where2.test b/test/where2.test index 367eb0dfea..434a7bcd9a 100644 --- a/test/where2.test +++ b/test/where2.test @@ -322,9 +322,14 @@ do_test where2-6.3 { } {6 2 49 51 99 6 10000 10006 100 6 10201 10207 sort t1 *} do_test where2-6.4 { queryplan { - SELECT * FROM t1 WHERE w=99 OR +w=100 OR 6=w ORDER BY +w + SELECT *, '|' FROM t1 WHERE w=99 OR +w=100 OR 6=w ORDER BY +w } -} {6 2 49 51 99 6 10000 10006 100 6 10201 10207 sort t1 *} +} {6 2 49 51 | 99 6 10000 10006 | 100 6 10201 10207 | sort t1 *} +do_test where2-6.5 { + queryplan { + SELECT *, '|' FROM t1 WHERE w=99 OR y=10201 OR 6=w ORDER BY +w + } +} {6 2 49 51 | 99 6 10000 10006 | 100 6 10201 10207 | sort t1 *} set ::idx {} ifcapable subquery {set ::idx i1zyx} From ecb5fedb3f7681fe963f74911e946414e0f65212 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Aug 2015 03:33:50 +0000 Subject: [PATCH 45/54] Enhance the json_insert(), json_replace(), and json_set() functions with the ability to add JSON instead of text if the argument is text and if the PATH begins with '$$' instead of just '$'. FossilOrigin-Name: 44f103d8862abc2d5613bac04dc2ea8e625b1f40 --- ext/misc/json1.c | 36 ++++++++++++++++++++++++++++-------- manifest | 14 +++++++------- manifest.uuid | 2 +- test/json101.test | 12 ++++++++++++ 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index cd4531bccb..dba8ea6daf 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -80,6 +80,7 @@ static const char * const jsonType[] = { #define JNODE_REMOVE 0x04 /* Do not output */ #define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ #define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_JSON 0x20 /* Treat REPLACE as JSON text */ /* A single node of parsed JSON @@ -238,7 +239,8 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ */ static void jsonAppendValue( JsonString *p, /* Append to this JSON string */ - sqlite3_value *pValue /* Value to append */ + sqlite3_value *pValue, /* Value to append */ + u8 textIsJson /* Try to treat text values as JSON */ ){ switch( sqlite3_value_type(pValue) ){ case SQLITE_NULL: { @@ -255,7 +257,11 @@ static void jsonAppendValue( case SQLITE_TEXT: { const char *z = (const char*)sqlite3_value_text(pValue); u32 n = (u32)sqlite3_value_bytes(pValue); - jsonAppendString(p, z, n); + if( textIsJson ){ + jsonAppendRaw(p, z, n); + }else{ + jsonAppendString(p, z, n); + } break; } default: { @@ -355,7 +361,8 @@ static void jsonRenderNode( if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ if( pNode[j].jnFlags & JNODE_REPLACE ){ jsonAppendSeparator(pOut); - jsonAppendValue(pOut, aReplace[pNode[j].iVal]); + jsonAppendValue(pOut, aReplace[pNode[j].iVal], + (pNode[j].jnFlags & JNODE_JSON)!=0); } }else{ jsonAppendSeparator(pOut); @@ -380,7 +387,8 @@ static void jsonRenderNode( jsonRenderNode(&pNode[j], pOut, aReplace); jsonAppendChar(pOut, ':'); if( pNode[j+1].jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); + jsonAppendValue(pOut, aReplace[pNode[j+1].iVal], + (pNode[j+1].jnFlags & JNODE_JSON)!=0); }else{ jsonRenderNode(&pNode[j+1], pOut, aReplace); } @@ -993,7 +1001,7 @@ static void jsonArrayFunc( jsonAppendChar(&jx, '['); for(i=0; ijnFlags |= JNODE_REPLACE; + pNode->jnFlags &= ~JNODE_JSON; + pNode->jnFlags |= jnFlags; pNode->iVal = i+1; } } @@ -1216,16 +1230,22 @@ static void jsonSetFunc( if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ for(i=1; i<(u32)argc; i+=2){ + u8 jnFlags = JNODE_REPLACE; zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) continue; if( zPath[0]!='$' ) continue; + if( zPath[1]=='$' ){ + zPath++; + jnFlags = JNODE_REPLACE|JNODE_JSON; + } bApnd = 0; pNode = jsonLookup(&x, 0, &zPath[1], &bApnd); if( x.oom ){ sqlite3_result_error_nomem(ctx); goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ - pNode->jnFlags |= JNODE_REPLACE; + pNode->jnFlags &= ~JNODE_JSON; + pNode->jnFlags |= jnFlags; pNode->iVal = i+1; } } diff --git a/manifest b/manifest index 4c7e7c0346..3ca9f5da3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sOR-optimization\sso\sthat\sit\salways\signores\ssubplans\sthat\sdo\snot\nuse\san\sindex. -D 2015-08-27T23:18:55.309 +C Enhance\sthe\sjson_insert(),\sjson_replace(),\sand\sjson_set()\sfunctions\swith\sthe\nability\sto\sadd\sJSON\sinstead\sof\stext\sif\sthe\sargument\sis\stext\sand\sif\sthe\sPATH\nbegins\swith\s'$$'\sinstead\sof\sjust\s'$'. +D 2015-08-28T03:33:50.087 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 541004e47235cefc2843ab03c100517452931913 +F ext/misc/json1.c e0aeaa8b2f374d06ca19081afbd70b271731b130 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -810,7 +810,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test 950ed4e8deb8ad4c10bd4fbc858eb54143de9867 +F test/json101.test 5dfb181790c123123c8e7981d4d3c941b6cc8af4 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cbc3c9a8bf169ae0b21f26855038502c6cc25cfe -R 3aef897e420a61a50263b8e461848e83 +P 66f92a16866e5825363636b9cc4b8f9b29d9e84d +R 8075e39a7f0130f58ac8ab5ea1825d8f U drh -Z b614bfef572643d9445d8b8a4d103409 +Z 9db1096145751bb3222f3dab3f132c33 diff --git a/manifest.uuid b/manifest.uuid index a3b504350e..d0af0aaaaf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -66f92a16866e5825363636b9cc4b8f9b29d9e84d \ No newline at end of file +44f103d8862abc2d5613bac04dc2ea8e625b1f40 \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index 1d788a6beb..752cd1b177 100644 --- a/test/json101.test +++ b/test/json101.test @@ -50,5 +50,17 @@ do_catchsql_test json1-2.4 { SELECT json_object('a',1,'b',x'abcd'); } {1 {JSON cannot hold BLOB values}} +do_execsql_test json1-3.1 { + SELECT json_replace('{"a":1,"b":2}','$.a','[3,4,5]'); +} {{{"a":"[3,4,5]","b":2}}} +do_execsql_test json1-3.2 { + SELECT json_replace('{"a":1,"b":2}','$$.a','[3,4,5]'); +} {{{"a":[3,4,5],"b":2}}} +do_execsql_test json1-3.3 { + SELECT json_type(json_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b'); +} {text} +do_execsql_test json1-3.4 { + SELECT json_type(json_set('{"a":1,"b":2}','$$.b','{"x":3,"y":4}'),'$.b'); +} {object} finish_test From f6ec8d4f4cdd858c052f09091f2bac706ec9cde4 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Aug 2015 03:48:04 +0000 Subject: [PATCH 46/54] Add the json_check() function, which returns its argument if the argument is well-formed JSON or which throws an error otherwise. FossilOrigin-Name: 64abb65d4df11e5b3bcc4afc8e7c18e907c6080a --- ext/misc/json1.c | 23 +++++++++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- test/json101.test | 7 +++++++ 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index dba8ea6daf..72ba33be0f 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -1049,6 +1049,28 @@ static void jsonArrayLengthFunc( if( !x.oom ) sqlite3_result_int64(ctx, n); } +/* +** json_check(JSON) +** +** Check the JSON argument to verify that it is well-formed. Return a +** compacted version of the argument (with white-space removed) if the +** argument is well-formed. Through an error if the argument is not +** correctly formatted JSON. +*/ +static void jsonCheckFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + return; + } + jsonReturn(x.aNode, ctx, argv); + jsonParseReset(&x); +} + /* ** json_extract(JSON, PATH) ** @@ -1780,6 +1802,7 @@ int sqlite3_json_init( { "json_array", -1, 0, jsonArrayFunc }, { "json_array_length", 1, 0, jsonArrayLengthFunc }, { "json_array_length", 2, 0, jsonArrayLengthFunc }, + { "json_check", 1, 0, jsonCheckFunc }, { "json_extract", 2, 0, jsonExtractFunc }, { "json_insert", -1, 0, jsonSetFunc }, { "json_object", -1, 0, jsonObjectFunc }, diff --git a/manifest b/manifest index 3ca9f5da3e..d07b325859 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sjson_insert(),\sjson_replace(),\sand\sjson_set()\sfunctions\swith\sthe\nability\sto\sadd\sJSON\sinstead\sof\stext\sif\sthe\sargument\sis\stext\sand\sif\sthe\sPATH\nbegins\swith\s'$$'\sinstead\sof\sjust\s'$'. -D 2015-08-28T03:33:50.087 +C Add\sthe\sjson_check()\sfunction,\swhich\sreturns\sits\sargument\sif\sthe\sargument\nis\swell-formed\sJSON\sor\swhich\sthrows\san\serror\sotherwise. +D 2015-08-28T03:48:04.807 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c e0aeaa8b2f374d06ca19081afbd70b271731b130 +F ext/misc/json1.c 0b2bf3f2c3786fd6873b5f9a89023912c28ac889 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -810,7 +810,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test 5dfb181790c123123c8e7981d4d3c941b6cc8af4 +F test/json101.test 420fe3917c9a6a0fb7f967f1f641054509acd0e2 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 66f92a16866e5825363636b9cc4b8f9b29d9e84d -R 8075e39a7f0130f58ac8ab5ea1825d8f +P 44f103d8862abc2d5613bac04dc2ea8e625b1f40 +R 204205d2daf18b22608467b1b10a8a77 U drh -Z 9db1096145751bb3222f3dab3f132c33 +Z 4f534e8322ceee4fbcec765d09d17add diff --git a/manifest.uuid b/manifest.uuid index d0af0aaaaf..49f83f8b9f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -44f103d8862abc2d5613bac04dc2ea8e625b1f40 \ No newline at end of file +64abb65d4df11e5b3bcc4afc8e7c18e907c6080a \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index 752cd1b177..2b9cafa00f 100644 --- a/test/json101.test +++ b/test/json101.test @@ -63,4 +63,11 @@ do_execsql_test json1-3.4 { SELECT json_type(json_set('{"a":1,"b":2}','$$.b','{"x":3,"y":4}'),'$.b'); } {object} +do_execsql_test json1-4.1 { + SELECT json_check(' [ 1, 2] '); +} {[1,2]} +do_catchsql_test json1-4.2 { + SELECT json_check(' [ 1, 2 '); +} {1 {malformed JSON}} + finish_test From 2798f0b54bf724f4919b1926cea9d1c47a74e01c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 28 Aug 2015 16:41:45 +0000 Subject: [PATCH 47/54] Fix compiler warnings in rbu code. FossilOrigin-Name: 0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 --- ext/rbu/sqlite3rbu.c | 11 ++++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 7c7480bcfc..d76d2f4ccd 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -489,7 +489,7 @@ static int rbuDeltaApply( /* ERROR: copy exceeds output file size */ return -1; } - if( ofst+cnt > lenSrc ){ + if( (int)(ofst+cnt) > lenSrc ){ /* ERROR: copy extends past end of input */ return -1; } @@ -504,7 +504,7 @@ static int rbuDeltaApply( /* ERROR: insert command gives an output larger than predicted */ return -1; } - if( cnt>lenDelta ){ + if( (int)cnt>lenDelta ){ /* ERROR: insert count exceeds size of delta */ return -1; } @@ -1117,7 +1117,7 @@ static void rbuTableType( } rbuTableType_end: { - int i; + unsigned int i; for(i=0; irc==SQLITE_OK ){ int i; - if( strlen(zMask)!=pIter->nTblCol ){ + if( (int)strlen(zMask)!=pIter->nTblCol ){ rbuBadControlError(p); }else{ const char *zSep = ""; @@ -3680,7 +3680,8 @@ static int rbuVfsOpen( rbuVfsShmMap, /* xShmMap */ rbuVfsShmLock, /* xShmLock */ rbuVfsShmBarrier, /* xShmBarrier */ - rbuVfsShmUnmap /* xShmUnmap */ + rbuVfsShmUnmap, /* xShmUnmap */ + 0, 0 /* xFetch, xUnfetch */ }; rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs; sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs; diff --git a/manifest b/manifest index ae51413a30..f4ea3f942d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\ssearching\sthe\swal\sfile\sfor\sa\sframe,\sdo\snot\ssearch\sthat\spart\sthat\swas\salready\scheckpointed\swhen\sthe\stransaction\swas\sopened. -D 2015-08-28T16:18:45.828 +C Fix\scompiler\swarnings\sin\srbu\scode. +D 2015-08-28T16:41:45.530 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -226,7 +226,7 @@ F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c 1650e682b3568db0ed97ff2c7ba5d1c8ea060a84 +F ext/rbu/sqlite3rbu.c 4ba82bd850aa012f73c31dd242d570f18c9cc35a F ext/rbu/sqlite3rbu.h 5357f070cd8c0bcad459b620651ec1656859e4d0 F ext/rbu/test_rbu.c 2a3652241fa45d5eaa141775e4ae68c1d3582c03 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 64abb65d4df11e5b3bcc4afc8e7c18e907c6080a ab93024da7bd577d8850b417aff8597a8b3807fc -R 3ba2e96fe96df838d1a765a1a1eb0bee +P a84cf4f5d326270a61faf4ff867260f2dd1e68a6 +R 25627e3633787096f1c6a6ac573b9337 U dan -Z e63e6f4df8bd62c2964d64d0b85672a7 +Z 8fac2bb28cf0b676d514df6920f608e8 diff --git a/manifest.uuid b/manifest.uuid index f8e9087de2..453c3b2f4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a84cf4f5d326270a61faf4ff867260f2dd1e68a6 \ No newline at end of file +0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 \ No newline at end of file From f2df7e71d6f0c970874552b67bbba113fc069e22 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Aug 2015 20:07:40 +0000 Subject: [PATCH 48/54] Back out the json_check() routine. Instead, throw an error if the input to a json function (other than json_valid()) is not valid JSON. FossilOrigin-Name: dc9ce7b18cbe23d065317757234ef9fb8792da7a --- ext/misc/json1.c | 81 ++++++++++++++++++++++------------------------- manifest | 16 +++++----- manifest.uuid | 2 +- test/json101.test | 7 ---- 4 files changed, 47 insertions(+), 59 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 72ba33be0f..a8b9ed539d 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -21,7 +21,9 @@ ** This implementation parses JSON text at 250 MB/s, so it is hard to see ** how JSONB might improve on that.) */ +#if !defined(_SQLITEINT_H_) #include "sqlite3ext.h" +#endif SQLITE_EXTENSION_INIT1 #include #include @@ -405,6 +407,20 @@ static void jsonRenderNode( } } +/* +** Return a JsonNode and all its descendents as a JSON string. +*/ +static void jsonReturnJson( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ + JsonString s; + jsonInit(&s, pCtx); + jsonRenderNode(pNode, &s, aReplace); + jsonResult(&s); +} + /* ** Make the JsonNode the return value of the function. */ @@ -509,10 +525,7 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - JsonString s; - jsonInit(&s, pCtx); - jsonRenderNode(pNode, &s, aReplace); - jsonResult(&s); + jsonReturnJson(pNode, pCtx, aReplace); break; } } @@ -717,7 +730,13 @@ static int jsonParse( if( zJson[i] ) i = -1; } if( i<0 ){ - if( pParse->oom && pCtx!=0 ) sqlite3_result_error_nomem(pCtx); + if( pCtx!=0 ){ + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + }else{ + sqlite3_result_error(pCtx, "malformed JSON", -1); + } + } jsonParseReset(pParse); return 1; } @@ -960,7 +979,7 @@ static void jsonTest1Func( ){ JsonParse x; /* The parse */ if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonReturn(x.aNode, ctx, 0); + jsonReturnJson(x.aNode, ctx, 0); jsonParseReset(&x); } @@ -1033,42 +1052,19 @@ static void jsonArrayLengthFunc( }else{ zPath = 0; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]))==0 ){ - if( x.nNode ){ - JsonNode *pNode = x.aNode; - if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); - if( pNode->eType==JSON_ARRAY ){ - assert( (pNode->jnFlags & JNODE_APPEND)==0 ); - for(i=1; i<=pNode->n; n++){ - i += jsonNodeSize(&pNode[i]); - } + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( x.nNode ){ + JsonNode *pNode = x.aNode; + if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); + if( pNode->eType==JSON_ARRAY ){ + assert( (pNode->jnFlags & JNODE_APPEND)==0 ); + for(i=1; i<=pNode->n; n++){ + i += jsonNodeSize(&pNode[i]); } } - jsonParseReset(&x); } - if( !x.oom ) sqlite3_result_int64(ctx, n); -} - -/* -** json_check(JSON) -** -** Check the JSON argument to verify that it is well-formed. Return a -** compacted version of the argument (with white-space removed) if the -** argument is well-formed. Through an error if the argument is not -** correctly formatted JSON. -*/ -static void jsonCheckFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ){ - sqlite3_result_error(ctx, "malformed JSON", -1); - return; - } - jsonReturn(x.aNode, ctx, argv); jsonParseReset(&x); + sqlite3_result_int64(ctx, n); } /* @@ -1166,7 +1162,7 @@ static void jsonRemoveFunc( if( pNode ) pNode->jnFlags |= JNODE_REMOVE; } if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturn(x.aNode, ctx, 0); + jsonReturnJson(x.aNode, ctx, 0); } } jsonParseReset(&x); @@ -1214,7 +1210,7 @@ static void jsonReplaceFunc( if( x.aNode[0].jnFlags & JNODE_REPLACE ){ sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); }else{ - jsonReturn(x.aNode, ctx, argv); + jsonReturnJson(x.aNode, ctx, argv); } } jsonParseReset(&x); @@ -1274,7 +1270,7 @@ static void jsonSetFunc( if( x.aNode[0].jnFlags & JNODE_REPLACE ){ sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); }else{ - jsonReturn(x.aNode, ctx, argv); + jsonReturnJson(x.aNode, ctx, argv); } } jsonSetDone: @@ -1328,7 +1324,7 @@ static void jsonValidFunc( JsonParse x; /* The parse */ int rc = 0; - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]))==0 + if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 && x.nNode>0 ){ rc = 1; @@ -1802,7 +1798,6 @@ int sqlite3_json_init( { "json_array", -1, 0, jsonArrayFunc }, { "json_array_length", 1, 0, jsonArrayLengthFunc }, { "json_array_length", 2, 0, jsonArrayLengthFunc }, - { "json_check", 1, 0, jsonCheckFunc }, { "json_extract", 2, 0, jsonExtractFunc }, { "json_insert", -1, 0, jsonSetFunc }, { "json_object", -1, 0, jsonObjectFunc }, diff --git a/manifest b/manifest index f4ea3f942d..630423411b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\sin\srbu\scode. -D 2015-08-28T16:41:45.530 +C Back\sout\sthe\sjson_check()\sroutine.\s\sInstead,\sthrow\san\serror\sif\sthe\sinput\sto\na\sjson\sfunction\s(other\sthan\sjson_valid())\sis\snot\svalid\sJSON. +D 2015-08-28T20:07:40.320 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 0b2bf3f2c3786fd6873b5f9a89023912c28ac889 +F ext/misc/json1.c aa344845c2c211e8a7fd57ccd92901506dacdf5a F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -810,7 +810,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test 420fe3917c9a6a0fb7f967f1f641054509acd0e2 +F test/json101.test 5dfb181790c123123c8e7981d4d3c941b6cc8af4 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a84cf4f5d326270a61faf4ff867260f2dd1e68a6 -R 25627e3633787096f1c6a6ac573b9337 -U dan -Z 8fac2bb28cf0b676d514df6920f608e8 +P 0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 +R 72483a04f1004ed82fe324bdb4440aeb +U drh +Z 66ee5efb60acb350500ed2a1605628ef diff --git a/manifest.uuid b/manifest.uuid index 453c3b2f4d..b5b08b89f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 \ No newline at end of file +dc9ce7b18cbe23d065317757234ef9fb8792da7a \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index 2b9cafa00f..752cd1b177 100644 --- a/test/json101.test +++ b/test/json101.test @@ -63,11 +63,4 @@ do_execsql_test json1-3.4 { SELECT json_type(json_set('{"a":1,"b":2}','$$.b','{"x":3,"y":4}'),'$.b'); } {object} -do_execsql_test json1-4.1 { - SELECT json_check(' [ 1, 2] '); -} {[1,2]} -do_catchsql_test json1-4.2 { - SELECT json_check(' [ 1, 2 '); -} {1 {malformed JSON}} - finish_test From a771402e558b3adcb201a84cbefe1ed5b7bd8076 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Aug 2015 00:54:49 +0000 Subject: [PATCH 49/54] Change the json1.c module so that it throws an error if any of the JSON selector paths are malformed. FossilOrigin-Name: 3aa0855fd463076fc3277f1d9fe00d2f30e6b449 --- ext/misc/json1.c | 204 +++++++++++++++++++++++++++++++++-------------- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 150 insertions(+), 68 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index a8b9ed539d..b4506c773f 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -108,6 +108,7 @@ struct JsonParse { const char *zJson; /* Original JSON string */ u32 *aUp; /* Index of parent of each node */ u8 oom; /* Set to true if out of memory */ + u8 nErr; /* Number of errors seen */ }; /************************************************************************** @@ -786,7 +787,7 @@ static int jsonParseFindParents(JsonParse *pParse){ } /* forward declaration */ -static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*); +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); /* ** Search along zPath to find the node specified. Return a pointer @@ -797,11 +798,12 @@ static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*); ** possible to do so and if no existing node corresponds to zPath. If ** new nodes are appended *pApnd is set to 1. */ -static JsonNode *jsonLookup( +static JsonNode *jsonLookupStep( JsonParse *pParse, /* The JSON to search */ u32 iRoot, /* Begin the search at this node */ const char *zPath, /* The path to search */ - int *pApnd /* Append nodes to complete path if not NULL */ + int *pApnd, /* Append nodes to complete path if not NULL */ + const char **pzErr /* Make *pzErr point to any syntax error in zPath */ ){ u32 i, j, nKey; const char *zKey; @@ -820,14 +822,17 @@ static JsonNode *jsonLookup( for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} nKey = i; } - if( nKey==0 ) return 0; + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } j = 1; for(;;){ while( j<=pRoot->n ){ if( pRoot[j].n==nKey+2 && strncmp(&pRoot[j].u.zJContent[1],zKey,nKey)==0 ){ - return jsonLookup(pParse, iRoot+j+1, &zPath[i], pApnd); + return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); } j++; j += jsonNodeSize(&pRoot[j]); @@ -843,7 +848,7 @@ static JsonNode *jsonLookup( iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); iLabel = jsonParseAddNode(pParse, JSON_STRING, i, zPath); zPath += i; - pNode = jsonLookupAppend(pParse, zPath, pApnd); + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); if( pParse->oom ) return 0; if( pNode ){ pRoot = &pParse->aNode[iRoot]; @@ -861,7 +866,10 @@ static JsonNode *jsonLookup( i = i*10 + zPath[0] - '0'; zPath++; } - if( zPath[0]!=']' ) return 0; + if( zPath[0]!=']' ){ + *pzErr = zPath; + return 0; + } zPath++; j = 1; for(;;){ @@ -875,13 +883,13 @@ static JsonNode *jsonLookup( j = 1; } if( j<=pRoot->n ){ - return jsonLookup(pParse, iRoot+j, zPath, pApnd); + return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); } if( i==0 && pApnd ){ u32 iStart; JsonNode *pNode; iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); - pNode = jsonLookupAppend(pParse, zPath, pApnd); + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); if( pParse->oom ) return 0; if( pNode ){ pRoot = &pParse->aNode[iRoot]; @@ -890,6 +898,8 @@ static JsonNode *jsonLookup( } return pNode; } + }else if( zPath[0]!=0 ){ + *pzErr = zPath; } return 0; } @@ -901,7 +911,8 @@ static JsonNode *jsonLookup( static JsonNode *jsonLookupAppend( JsonParse *pParse, /* Append content to the JSON parse */ const char *zPath, /* Description of content to append */ - int *pApnd /* Set this flag to 1 */ + int *pApnd, /* Set this flag to 1 */ + const char **pzErr /* Make this point to any syntax error */ ){ *pApnd = 1; if( zPath[0]==0 ){ @@ -916,9 +927,76 @@ static JsonNode *jsonLookupAppend( return 0; } if( pParse->oom ) return 0; - return jsonLookup(pParse, pParse->nNode-1, zPath, pApnd); + return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); } +/* +** Return the text of a syntax error message on a JSON path. Space is +** obtained from sqlite3_malloc(). +*/ +static char *jsonPathSyntaxError(const char *zErr){ + return sqlite3_mprintf("JSON path error near '%q'", zErr); +} + +/* +** Do a node lookup using zPath. Return a pointer to the node on success. +** Return NULL if not found or if there is an error. +** +** On an error, write an error message into pCtx and increment the +** pParse->nErr counter. +** +** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if +** nodes are appended. +** +** If the path starts with $$ then set *pFlags to JNODE_REPLACE|JNODE_JSON +** as a single to the caller that the input text to be inserted should be +** interpreted as JSON rather than as ordinary text. +*/ +static JsonNode *jsonLookup( + JsonParse *pParse, /* The JSON to search */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + sqlite3_context *pCtx, /* Report errors here, if not NULL */ + u8 *pFlags /* Write JNODE_REPLACE or _REPLACE|_JSON here */ +){ + const char *zErr = 0; + JsonNode *pNode = 0; + u8 fg = JNODE_REPLACE; + + if( zPath==0 ) return 0; + if( zPath[0]!='$' ){ + zErr = zPath; + goto lookup_err; + } + zPath++; + if( zPath[0]=='$' ){ + if( pFlags==0 ){ + zErr = zPath; + goto lookup_err; + } + zPath++; + fg = JNODE_REPLACE|JNODE_JSON; + } + if( pFlags ) *pFlags = fg; + pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); + return pNode; + +lookup_err: + pParse->nErr++; + if( zErr!=0 && pCtx!=0 ){ + char *z = jsonPathSyntaxError(zErr); + if( z ){ + sqlite3_result_error(pCtx, z, -1); + sqlite3_free(z); + }else{ + sqlite3_result_error_nomem(pCtx); + } + } + if( pFlags ) *pFlags = fg; + return 0; +} + + /* ** Report the wrong number of arguments for json_insert(), json_replace() ** or json_set(). @@ -933,6 +1011,7 @@ static void jsonWrongNumArgs( sqlite3_free(zMsg); } + /**************************************************************************** ** SQL functions used for testing and debugging ****************************************************************************/ @@ -1042,29 +1121,27 @@ static void jsonArrayLengthFunc( JsonParse x; /* The parse */ sqlite3_int64 n = 0; u32 i; - const char *zPath; - if( argc==2 ){ - zPath = (const char*)sqlite3_value_text(argv[1]); - if( zPath==0 ) return; - if( zPath[0]!='$' ) return; - zPath++; - }else{ - zPath = 0; - } if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ - JsonNode *pNode = x.aNode; - if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); - if( pNode->eType==JSON_ARRAY ){ + JsonNode *pNode; + if( argc==2 ){ + const char *zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(&x, zPath, 0, ctx, 0); + }else{ + pNode = x.aNode; + } + if( pNode==0 ){ + x.nErr = 1; + }else if( pNode->eType==JSON_ARRAY ){ assert( (pNode->jnFlags & JNODE_APPEND)==0 ); for(i=1; i<=pNode->n; n++){ i += jsonNodeSize(&pNode[i]); } } } + if( x.nErr==0 ) sqlite3_result_int64(ctx, n); jsonParseReset(&x); - sqlite3_result_int64(ctx, n); } /* @@ -1083,11 +1160,8 @@ static void jsonExtractFunc( const char *zPath; assert( argc==2 ); zPath = (const char*)sqlite3_value_text(argv[1]); - if( zPath==0 ) return; - if( zPath[0]!='$' ) return; - zPath++; if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - pNode = jsonLookup(&x, 0, zPath, 0); + pNode = jsonLookup(&x, zPath, 0, ctx, 0); if( pNode ){ jsonReturn(pNode, ctx, 0); } @@ -1156,15 +1230,16 @@ static void jsonRemoveFunc( if( x.nNode ){ for(i=1; i<(u32)argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); - if( zPath==0 ) continue; - if( zPath[0]!='$' ) continue; - pNode = jsonLookup(&x, 0, &zPath[1], 0); + if( zPath==0 ) goto remove_done; + pNode = jsonLookup(&x, zPath, 0, ctx, 0); + if( x.nErr ) goto remove_done; if( pNode ) pNode->jnFlags |= JNODE_REMOVE; } if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ jsonReturnJson(x.aNode, ctx, 0); } } +remove_done: jsonParseReset(&x); } @@ -1194,13 +1269,8 @@ static void jsonReplaceFunc( for(i=1; i<(u32)argc; i+=2){ u8 jnFlags = JNODE_REPLACE; zPath = (const char*)sqlite3_value_text(argv[i]); - if( zPath==0 ) continue; - if( zPath[0]!='$' ) continue; - if( zPath[1]=='$' ){ - zPath++; - jnFlags = JNODE_REPLACE|JNODE_JSON; - } - pNode = jsonLookup(&x, 0, &zPath[1], 0); + pNode = jsonLookup(&x, zPath, 0, ctx, &jnFlags); + if( x.nErr ) goto replace_err; if( pNode ){ pNode->jnFlags &= ~JNODE_JSON; pNode->jnFlags |= jnFlags; @@ -1213,6 +1283,7 @@ static void jsonReplaceFunc( jsonReturnJson(x.aNode, ctx, argv); } } +replace_err: jsonParseReset(&x); } @@ -1250,17 +1321,13 @@ static void jsonSetFunc( for(i=1; i<(u32)argc; i+=2){ u8 jnFlags = JNODE_REPLACE; zPath = (const char*)sqlite3_value_text(argv[i]); - if( zPath==0 ) continue; - if( zPath[0]!='$' ) continue; - if( zPath[1]=='$' ){ - zPath++; - jnFlags = JNODE_REPLACE|JNODE_JSON; - } bApnd = 0; - pNode = jsonLookup(&x, 0, &zPath[1], &bApnd); + pNode = jsonLookup(&x, zPath, &bApnd, ctx, &jnFlags); if( x.oom ){ sqlite3_result_error_nomem(ctx); goto jsonSetDone; + }else if( x.nErr ){ + goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ pNode->jnFlags &= ~JNODE_JSON; pNode->jnFlags |= jnFlags; @@ -1292,18 +1359,15 @@ static void jsonTypeFunc( JsonParse x; /* The parse */ const char *zPath; - if( argc==2 ){ - zPath = (const char*)sqlite3_value_text(argv[1]); - if( zPath==0 ) return; - if( zPath[0]!='$' ) return; - zPath++; - }else{ - zPath = 0; - } if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ - JsonNode *pNode = x.aNode; - if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); + JsonNode *pNode; + if( argc==2 ){ + zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(&x, zPath, 0, ctx, 0); + }else{ + pNode = x.aNode; + } if( pNode ){ sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); } @@ -1680,27 +1744,45 @@ static int jsonEachFilter( if( z==0 ) return SQLITE_OK; if( idxNum&2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); - if( zPath==0 || zPath[0]!='$' ) return SQLITE_OK; + if( zPath==0 ) return SQLITE_OK; + if( zPath[0]!='$' ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zPath); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + } } n = sqlite3_value_bytes(argv[0]); p->zJson = sqlite3_malloc64( n+1 ); if( p->zJson==0 ) return SQLITE_NOMEM; memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson) - || (p->bRecursive && jsonParseFindParents(&p->sParse)) - ){ + if( jsonParse(&p->sParse, 0, p->zJson) ){ + int rc = SQLITE_NOMEM; + if( p->sParse.oom==0 ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; + } jsonEachCursorReset(p); + return rc; + }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ + jsonEachCursorReset(p); + return SQLITE_NOMEM; }else{ JsonNode *pNode; if( idxNum==3 ){ + const char *zErr = 0; p->bRecursive = 0; n = sqlite3_value_bytes(argv[1]); p->zPath = sqlite3_malloc64( n+1 ); if( p->zPath==0 ) return SQLITE_NOMEM; memcpy(p->zPath, zPath, (size_t)n+1); - pNode = jsonLookup(&p->sParse, 0, p->zPath+1, 0); - if( pNode==0 ){ + pNode = jsonLookupStep(&p->sParse, 0, p->zPath+1, 0, &zErr); + if( p->sParse.nErr ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + }else if( pNode==0 ){ return SQLITE_OK; } }else{ diff --git a/manifest b/manifest index 630423411b..dda3e678a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Back\sout\sthe\sjson_check()\sroutine.\s\sInstead,\sthrow\san\serror\sif\sthe\sinput\sto\na\sjson\sfunction\s(other\sthan\sjson_valid())\sis\snot\svalid\sJSON. -D 2015-08-28T20:07:40.320 +C Change\sthe\sjson1.c\smodule\sso\sthat\sit\sthrows\san\serror\sif\sany\sof\sthe\nJSON\sselector\spaths\sare\smalformed. +D 2015-08-29T00:54:49.924 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c aa344845c2c211e8a7fd57ccd92901506dacdf5a +F ext/misc/json1.c 063bf62fd44a06aa06fd22854fff09679cbb855f F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 -R 72483a04f1004ed82fe324bdb4440aeb +P dc9ce7b18cbe23d065317757234ef9fb8792da7a +R 7515083280f6e706dffbefc132c1a59a U drh -Z 66ee5efb60acb350500ed2a1605628ef +Z f18bb212545a9a98b8ef3bc5262369aa diff --git a/manifest.uuid b/manifest.uuid index b5b08b89f4..19c1a06207 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc9ce7b18cbe23d065317757234ef9fb8792da7a \ No newline at end of file +3aa0855fd463076fc3277f1d9fe00d2f30e6b449 \ No newline at end of file From d1f0068d190a8d0de786033fa2618eb0d504ec42 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Aug 2015 16:02:37 +0000 Subject: [PATCH 50/54] Do not consider an empty string to be valid JSON. Add some additional JSON test cases. FossilOrigin-Name: fd19ff029f128f478f69910352a6f8b84262ce1d --- ext/misc/json1.c | 4 ++-- manifest | 14 ++++++------- manifest.uuid | 2 +- test/json101.test | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index b4506c773f..af20c691fa 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -690,7 +690,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ j++; c = pParse->zJson[j+1]; } - if( c<'0' || c>'0' ) return -1; + if( c<'0' || c>'9' ) return -1; continue; } break; @@ -730,7 +730,7 @@ static int jsonParse( while( isspace(zJson[i]) ) i++; if( zJson[i] ) i = -1; } - if( i<0 ){ + if( i<=0 ){ if( pCtx!=0 ){ if( pParse->oom ){ sqlite3_result_error_nomem(pCtx); diff --git a/manifest b/manifest index dda3e678a6..89d83b2549 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sjson1.c\smodule\sso\sthat\sit\sthrows\san\serror\sif\sany\sof\sthe\nJSON\sselector\spaths\sare\smalformed. -D 2015-08-29T00:54:49.924 +C Do\snot\sconsider\san\sempty\sstring\sto\sbe\svalid\sJSON.\s\sAdd\ssome\sadditional\nJSON\stest\scases. +D 2015-08-29T16:02:37.845 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 063bf62fd44a06aa06fd22854fff09679cbb855f +F ext/misc/json1.c 232a3125fc468e9075f569b1b543b797fd4e0f81 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -810,7 +810,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test 5dfb181790c123123c8e7981d4d3c941b6cc8af4 +F test/json101.test ef8fb3ac6a59b9a435c9fec7a4eb6413ecf76531 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dc9ce7b18cbe23d065317757234ef9fb8792da7a -R 7515083280f6e706dffbefc132c1a59a +P 3aa0855fd463076fc3277f1d9fe00d2f30e6b449 +R 3daa1648ff31e492e4e517b1729e3cd5 U drh -Z f18bb212545a9a98b8ef3bc5262369aa +Z 4768dfaa3613e1e500aa484aea396edc diff --git a/manifest.uuid b/manifest.uuid index 19c1a06207..e78d4becc0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3aa0855fd463076fc3277f1d9fe00d2f30e6b449 \ No newline at end of file +fd19ff029f128f478f69910352a6f8b84262ce1d \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index 752cd1b177..be561094d5 100644 --- a/test/json101.test +++ b/test/json101.test @@ -63,4 +63,54 @@ do_execsql_test json1-3.4 { SELECT json_type(json_set('{"a":1,"b":2}','$$.b','{"x":3,"y":4}'),'$.b'); } {object} +# Per rfc7159, any JSON value is allowed at the top level, and whitespace +# is permitting before and/or after that value. +# +do_execsql_test json1-4.1 { + CREATE TABLE j1(x); + INSERT INTO j1(x) + VALUES('true'),('false'),('null'),('123'),('-234'),('34.5e+6'), + ('""'),('"\""'),('"\\"'),('"abcdefghijlmnopqrstuvwxyz"'), + ('[]'),('{}'),('[true,false,null,123,-234,34.5e+6,{},[]]'), + ('{"a":true,"b":{"c":false}}'); + SELECT * FROM j1 WHERE NOT json_valid(x); +} {} +do_execsql_test json1-4.2 { + SELECT * FROM j1 WHERE NOT json_valid(char(0x20,0x09,0x0a,0x0d)||x); +} {} +do_execsql_test json1-4.3 { + SELECT * FROM j1 WHERE NOT json_valid(x||char(0x20,0x09,0x0a,0x0d)); +} {} + +# But an empty string, or a string of pure whitespace is not valid JSON. +# +do_execsql_test json1-4.4 { + SELECT json_valid(''), json_valid(char(0x20,0x09,0x0a,0x0d)); +} {0 0} + +# json_remove() and similar functions with no edit operations return their +# input unchanged. +# +do_execsql_test json1-4.5 { + SELECT x FROM j1 WHERE json_remove(x)<>x; +} {} +do_execsql_test json1-4.6 { + SELECT x FROM j1 WHERE json_replace(x)<>x; +} {} +do_execsql_test json1-4.7 { + SELECT x FROM j1 WHERE json_set(x)<>x; +} {} +do_execsql_test json1-4.8 { + SELECT x FROM j1 WHERE json_insert(x)<>x; +} {} + +# json_extract(JSON,'$') will return objects and arrays without change. +# +do_execsql_test json-4.10 { + SELECT count(*) FROM j1 WHERE json_type(x) IN ('object','array'); + SELECT x FROM j1 + WHERE json_extract(x,'$')<>x + AND json_type(x) IN ('object','array'); +} {4} + finish_test From d29759288e1e744b42601aefb29ea120ae4d38e1 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Aug 2015 17:22:33 +0000 Subject: [PATCH 51/54] Fix the build with -DSQLITE_OMIT_VIRTUALTABLE. FossilOrigin-Name: 752918def7231f7846b3e985c9953a1cc825ab6b --- ext/misc/json1.c | 6 ++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index af20c691fa..aad5c57fdc 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -1397,6 +1397,7 @@ static void jsonValidFunc( sqlite3_result_int(ctx, rc); } +#ifndef SQLITE_OMIT_VIRTUALTABLE /**************************************************************************** ** The json_each virtual table ****************************************************************************/ @@ -1854,6 +1855,7 @@ static sqlite3_module jsonTreeModule = { 0, /* xRelease */ 0 /* xRollbackTo */ }; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ /**************************************************************************** ** The following routine is the only publically visible identifier in this @@ -1897,6 +1899,7 @@ int sqlite3_json_init( { "json_nodecount", 1, 0, jsonNodeCountFunc }, #endif }; +#ifndef SQLITE_OMIT_VIRTUALTABLE static const struct { const char *zName; sqlite3_module *pModule; @@ -1904,6 +1907,7 @@ int sqlite3_json_init( { "json_each", &jsonEachModule }, { "json_tree", &jsonTreeModule }, }; +#endif SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ for(i=0; idb, zName, zDbase); if( p==0 ){ const char *zMsg = isView ? "no such view" : "no such table"; -#ifndef SQLITE_OMIT_VIRTUAL_TABLE +#ifndef SQLITE_OMIT_VIRTUALTABLE /* If zName is the not the name of a table in the schema created using ** CREATE, then check to see if it is the name of an virtual table that ** can be an eponymous virtual table. */ From 20b3b6109ba2b5dcd8a19b17a11717cac49a9f9f Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Aug 2015 18:30:30 +0000 Subject: [PATCH 52/54] New test cases for the json1 extension. FossilOrigin-Name: daff4832af963f98bcc1d2c2f84bd815d384f850 --- manifest | 12 ++-- manifest.uuid | 2 +- test/json101.test | 179 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 6dcf2619a0..9b8023b059 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sbuild\swith\s-DSQLITE_OMIT_VIRTUALTABLE. -D 2015-08-29T17:22:33.763 +C New\stest\scases\sfor\sthe\sjson1\sextension. +D 2015-08-29T18:30:30.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -810,7 +810,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test ef8fb3ac6a59b9a435c9fec7a4eb6413ecf76531 +F test/json101.test 11535d8986184500f4c30cc2f0b154b4ab05cc4e F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fd19ff029f128f478f69910352a6f8b84262ce1d -R 9a4be85e9706d5dbc0e9b26aa7664d5f +P 752918def7231f7846b3e985c9953a1cc825ab6b +R 4e95d91aa0ecccfdfe103208bb14f5d9 U drh -Z 792dcf72d6ff70af8eda2770d6871186 +Z dd982e2b56c4c4ec66263dd341e5829e diff --git a/manifest.uuid b/manifest.uuid index 71b6d8e22e..7b3cfd163a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -752918def7231f7846b3e985c9953a1cc825ab6b \ No newline at end of file +daff4832af963f98bcc1d2c2f84bd815d384f850 \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index be561094d5..1a84a5fc5e 100644 --- a/test/json101.test +++ b/test/json101.test @@ -113,4 +113,183 @@ do_execsql_test json-4.10 { AND json_type(x) IN ('object','array'); } {4} +do_execsql_test json-5.1 { + CREATE TABLE j2(id INTEGER PRIMARY KEY, json, src); + INSERT INTO j2(id,json,src) + VALUES(1,'{ + "firstName": "John", + "lastName": "Smith", + "isAlive": true, + "age": 25, + "address": { + "streetAddress": "21 2nd Street", + "city": "New York", + "state": "NY", + "postalCode": "10021-3100" + }, + "phoneNumbers": [ + { + "type": "home", + "number": "212 555-1234" + }, + { + "type": "office", + "number": "646 555-4567" + } + ], + "children": [], + "spouse": null + }','https://en.wikipedia.org/wiki/JSON'); + INSERT INTO j2(id,json,src) + VALUES(2, '{ + "id": "0001", + "type": "donut", + "name": "Cake", + "ppu": 0.55, + "batters": + { + "batter": + [ + { "id": "1001", "type": "Regular" }, + { "id": "1002", "type": "Chocolate" }, + { "id": "1003", "type": "Blueberry" }, + { "id": "1004", "type": "Devil''s Food" } + ] + }, + "topping": + [ + { "id": "5001", "type": "None" }, + { "id": "5002", "type": "Glazed" }, + { "id": "5005", "type": "Sugar" }, + { "id": "5007", "type": "Powdered Sugar" }, + { "id": "5006", "type": "Chocolate with Sprinkles" }, + { "id": "5003", "type": "Chocolate" }, + { "id": "5004", "type": "Maple" } + ] + }','https://adobe.github.io/Spry/samples/data_region/JSONDataSetSample.html'); + INSERT INTO j2(id,json,src) + VALUES(3,'[ + { + "id": "0001", + "type": "donut", + "name": "Cake", + "ppu": 0.55, + "batters": + { + "batter": + [ + { "id": "1001", "type": "Regular" }, + { "id": "1002", "type": "Chocolate" }, + { "id": "1003", "type": "Blueberry" }, + { "id": "1004", "type": "Devil''s Food" } + ] + }, + "topping": + [ + { "id": "5001", "type": "None" }, + { "id": "5002", "type": "Glazed" }, + { "id": "5005", "type": "Sugar" }, + { "id": "5007", "type": "Powdered Sugar" }, + { "id": "5006", "type": "Chocolate with Sprinkles" }, + { "id": "5003", "type": "Chocolate" }, + { "id": "5004", "type": "Maple" } + ] + }, + { + "id": "0002", + "type": "donut", + "name": "Raised", + "ppu": 0.55, + "batters": + { + "batter": + [ + { "id": "1001", "type": "Regular" } + ] + }, + "topping": + [ + { "id": "5001", "type": "None" }, + { "id": "5002", "type": "Glazed" }, + { "id": "5005", "type": "Sugar" }, + { "id": "5003", "type": "Chocolate" }, + { "id": "5004", "type": "Maple" } + ] + }, + { + "id": "0003", + "type": "donut", + "name": "Old Fashioned", + "ppu": 0.55, + "batters": + { + "batter": + [ + { "id": "1001", "type": "Regular" }, + { "id": "1002", "type": "Chocolate" } + ] + }, + "topping": + [ + { "id": "5001", "type": "None" }, + { "id": "5002", "type": "Glazed" }, + { "id": "5003", "type": "Chocolate" }, + { "id": "5004", "type": "Maple" } + ] + } + ]','https://adobe.github.io/Spry/samples/data_region/JSONDataSetSample.html'); + SELECT count(*) FROM j2; +} {3} + +do_execsql_test json-5.2 { + SELECT id, json_valid(json), json_type(json), '|' FROM j2 ORDER BY id; +} {1 1 object | 2 1 object | 3 1 array |} + +ifcapable !vtab { + finish_test + return +} + +# fullkey is always the same as path+key (with appropriate formatting) +# +do_execsql_test json-5.3 { + SELECT j2.rowid, jx.rowid, fullkey, path, key + FROM j2, json_tree(j2.json) AS jx + WHERE fullkey!=(path || CASE WHEN typeof(key)=='integer' THEN '['||key||']' + ELSE '.'||key END); +} {} +do_execsql_test json-5.4 { + SELECT j2.rowid, jx.rowid, fullkey, path, key + FROM j2, json_each(j2.json) AS jx + WHERE fullkey!=(path || CASE WHEN typeof(key)=='integer' THEN '['||key||']' + ELSE '.'||key END); +} {} + + +# Verify that the json_each.json and json_tree.json output is always the +# same as input. +# +do_execsql_test json-5.5 { + SELECT j2.rowid, jx.rowid, fullkey, path, key + FROM j2, json_each(j2.json) AS jx + WHERE jx.json<>j2.json; +} {} +do_execsql_test json-5.6 { + SELECT j2.rowid, jx.rowid, fullkey, path, key + FROM j2, json_tree(j2.json) AS jx + WHERE jx.json<>j2.json; +} {} +do_execsql_test json-5.7 { + SELECT j2.rowid, jx.rowid, fullkey, path, key + FROM j2, json_each(j2.json) AS jx + WHERE jx.value<>jx.atom AND type NOT IN ('array','object'); +} {} +do_execsql_test json-5.8 { + SELECT j2.rowid, jx.rowid, fullkey, path, key + FROM j2, json_tree(j2.json) AS jx + WHERE jx.value<>jx.atom AND type NOT IN ('array','object'); +} {} + + + finish_test From 3be094df69c65ae6c8d0f124280f5c03062a6755 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Aug 2015 19:03:33 +0000 Subject: [PATCH 53/54] Link the json1 extension into the command-line shell by default. FossilOrigin-Name: 2e8e239cec5a12ac81cf62c0fbe94fb5713c31b1 --- Makefile.in | 6 +++--- Makefile.msc | 8 ++++---- main.mk | 6 +++--- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/shell.c | 7 +++++++ 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/Makefile.in b/Makefile.in index 8f3d6f7dfe..99af3e44e5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -566,9 +566,9 @@ libtclsqlite3.la: tclsqlite.lo libsqlite3.la -version-info "8:6:8" \ -avoid-version -sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h - $(LTLINK) $(READLINE_FLAGS) \ - -o $@ $(TOP)/src/shell.c libsqlite3.la \ +sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h $(TOP)/ext/misc/json1.c + $(LTLINK) $(READLINE_FLAGS) -DSQLITE_ENABLE_JSON1 -o $@ \ + $(TOP)/src/shell.c $(TOP)/ext/misc/json1.c libsqlite3.la \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h diff --git a/Makefile.msc b/Makefile.msc index 441c499df8..7b971aa2f1 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -390,9 +390,9 @@ CORE_LINK_OPTS = /DEF:sqlite3.def # !IFNDEF SHELL_COMPILE_OPTS !IF $(DYNAMIC_SHELL)!=0 -SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) +SHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) !ELSE -SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) +SHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 $(SHELL_CCONV_OPTS) !ENDIF !ENDIF @@ -1224,8 +1224,8 @@ libsqlite3.lib: $(LIBOBJ) libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS) -sqlite3.exe: $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h - $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c \ +sqlite3.exe: $(TOP)\src\shell.c $(TOP)\ext\misc\json1.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h + $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(TOP)\ext\misc\json1.c \ /link /pdb:sqlite3sh.pdb $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) sqldiff.exe: $(TOP)\tool\sqldiff.c sqlite3.c sqlite3.h diff --git a/main.mk b/main.mk index 9988fe2775..7cc09d6c8d 100644 --- a/main.mk +++ b/main.mk @@ -439,9 +439,9 @@ libsqlite3.a: $(LIBOBJ) $(AR) libsqlite3.a $(LIBOBJ) $(RANLIB) libsqlite3.a -sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h - $(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) \ - $(TOP)/src/shell.c \ +sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h $(TOP)/ext/misc/json1.c + $(TCCX) $(READLINE_FLAGS) -DSQLITE_ENABLE_JSON1 -o sqlite3$(EXE) \ + $(TOP)/src/shell.c $(TOP)/ext/misc/json1.c \ libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h diff --git a/manifest b/manifest index 9b8023b059..119ce7f47d 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C New\stest\scases\sfor\sthe\sjson1\sextension. -D 2015-08-29T18:30:30.457 +C Link\sthe\sjson1\sextension\sinto\sthe\scommand-line\sshell\sby\sdefault. +D 2015-08-29T19:03:33.633 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in e2218eb228374422969de7b1680eda6864affcef +F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 10af19cc089862481d49b347acd99c02635ddc49 +F Makefile.msc b268d8be2e800b9d35f074b1ed6b2f698deebdd6 F Makefile.vxworks e1b65dea203f054e71653415bd8f96dcaed47858 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION ccfc4d1576dbfdeece0a4372a2e6a2e37d3e7975 @@ -258,7 +258,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 04840e8277ab5159af16172eafd214dae7cffff5 +F main.mk 8da13ed011a7ae19450b7554910ff4afa3bd22b7 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -338,7 +338,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c f2ef256786a6435efddd64a632fea89c8be62215 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c b52c80f2b1bdb62491f9ce40eea0c5f80c78d105 -F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 +F src/shell.c bbe2bab590b7dd04dd8f9119d4473cb8c52906e3 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 752918def7231f7846b3e985c9953a1cc825ab6b -R 4e95d91aa0ecccfdfe103208bb14f5d9 +P daff4832af963f98bcc1d2c2f84bd815d384f850 +R 082d6d0f70d8c030232efb6a589595f2 U drh -Z dd982e2b56c4c4ec66263dd341e5829e +Z 988619e0978b399609ee7513adf0867a diff --git a/manifest.uuid b/manifest.uuid index 7b3cfd163a..d0eeda427c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -daff4832af963f98bcc1d2c2f84bd815d384f850 \ No newline at end of file +2e8e239cec5a12ac81cf62c0fbe94fb5713c31b1 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index c289998a7b..e04fe784a8 100644 --- a/src/shell.c +++ b/src/shell.c @@ -4619,6 +4619,13 @@ int SQLITE_CDECL main(int argc, char **argv){ } data.out = stdout; +#ifdef SQLITE_ENABLE_JSON1 + { + extern int sqlite3_json_init(sqlite3*); + sqlite3_auto_extension((void(*)(void))sqlite3_json_init); + } +#endif + /* Go ahead and open the database file if it already exists. If the ** file does not exist, delay opening it. This prevents empty database ** files from being created if a user mistypes the database name argument From 3ad93bba84bf191bfe75facd6680c4dfa4b3b827 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Aug 2015 19:41:45 +0000 Subject: [PATCH 54/54] Enhance the json_extract() function so that if given multiple PATH arguments it will return a JSON array with all of the answers. Also update comments within the json1 extension to reflect stricter interpretation of JSON and PATH arguments. FossilOrigin-Name: 1da60c3dda4254620052a83c853c2d2b6dd5009f --- ext/misc/json1.c | 58 +++++++++++++++++++++++++++++++++--------------- manifest | 12 +++++----- manifest.uuid | 2 +- 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index aad5c57fdc..5df7551dec 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -1145,10 +1145,12 @@ static void jsonArrayLengthFunc( } /* -** json_extract(JSON, PATH) +** json_extract(JSON, PATH, ...) ** -** Return the element described by PATH. Return NULL if JSON is not -** valid JSON or if there is no PATH element or if PATH is malformed. +** Return the element described by PATH. Return NULL if there is no +** PATH element. If there are multiple PATHs, then return a JSON array +** with the result from each path. Throw an error if the JSON or any PATH +** is malformed. */ static void jsonExtractFunc( sqlite3_context *ctx, @@ -1158,13 +1160,33 @@ static void jsonExtractFunc( JsonParse x; /* The parse */ JsonNode *pNode; const char *zPath; - assert( argc==2 ); - zPath = (const char*)sqlite3_value_text(argv[1]); + JsonString jx; + int i; + + if( argc<2 ) return; if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - pNode = jsonLookup(&x, zPath, 0, ctx, 0); - if( pNode ){ - jsonReturn(pNode, ctx, 0); + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '['); + for(i=1; i2 ){ + jsonAppendSeparator(&jx); + if( pNode ){ + jsonRenderNode(pNode, &jx, 0); + }else{ + jsonAppendRaw(&jx, "null", 4); + } + }else if( pNode ){ + jsonReturn(pNode, ctx, 0); + } } + if( argc>2 && i==argc ){ + jsonAppendChar(&jx, ']'); + jsonResult(&jx); + } + jsonReset(&jx); jsonParseReset(&x); } @@ -1211,9 +1233,8 @@ static void jsonObjectFunc( /* ** json_remove(JSON, PATH, ...) ** -** Remove the named elements from JSON and return the result. Ill-formed -** PATH arguments are silently ignored. If JSON is ill-formed, then NULL -** is returned. +** Remove the named elements from JSON and return the result. malformed +** JSON or PATH arguments result in an error. */ static void jsonRemoveFunc( sqlite3_context *ctx, @@ -1247,7 +1268,7 @@ remove_done: ** json_replace(JSON, PATH, VALUE, ...) ** ** Replace the value at PATH with VALUE. If PATH does not already exist, -** this routine is a no-op. If JSON is ill-formed, return NULL. +** this routine is a no-op. If JSON or PATH is malformed, throw an error. */ static void jsonReplaceFunc( sqlite3_context *ctx, @@ -1292,12 +1313,12 @@ replace_err: ** ** Set the value at PATH to VALUE. Create the PATH if it does not already ** exist. Overwrite existing values that do exist. -** If JSON is ill-formed, return NULL. +** If JSON or PATH is malformed, throw an error. ** ** json_insert(JSON, PATH, VALUE, ...) ** ** Create PATH and initialize it to VALUE. If PATH already exists, this -** routine is a no-op. If JSON is ill-formed, return NULL. +** routine is a no-op. If JSON or PATH is malformed, throw an error. */ static void jsonSetFunc( sqlite3_context *ctx, @@ -1348,8 +1369,8 @@ jsonSetDone: ** json_type(JSON) ** json_type(JSON, PATH) ** -** Return the top-level "type" of a JSON string. Return NULL if the -** input is not a well-formed JSON string. +** Return the top-level "type" of a JSON string. Throw an error if +** either the JSON or PATH inputs are not well-formed. */ static void jsonTypeFunc( sqlite3_context *ctx, @@ -1378,7 +1399,8 @@ static void jsonTypeFunc( /* ** json_valid(JSON) ** -** Return 1 if JSON is a valid JSON string. Return 0 otherwise. +** Return 1 if JSON is a well-formed JSON string according to RFC-7159. +** Return 0 otherwise. */ static void jsonValidFunc( sqlite3_context *ctx, @@ -1882,7 +1904,7 @@ int sqlite3_json_init( { "json_array", -1, 0, jsonArrayFunc }, { "json_array_length", 1, 0, jsonArrayLengthFunc }, { "json_array_length", 2, 0, jsonArrayLengthFunc }, - { "json_extract", 2, 0, jsonExtractFunc }, + { "json_extract", -1, 0, jsonExtractFunc }, { "json_insert", -1, 0, jsonSetFunc }, { "json_object", -1, 0, jsonObjectFunc }, { "json_remove", -1, 0, jsonRemoveFunc }, diff --git a/manifest b/manifest index 119ce7f47d..cb2ed48858 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Link\sthe\sjson1\sextension\sinto\sthe\scommand-line\sshell\sby\sdefault. -D 2015-08-29T19:03:33.633 +C Enhance\sthe\sjson_extract()\sfunction\sso\sthat\sif\sgiven\smultiple\sPATH\sarguments\nit\swill\sreturn\sa\sJSON\sarray\swith\sall\sof\sthe\sanswers.\s\sAlso\supdate\scomments\nwithin\sthe\sjson1\sextension\sto\sreflect\sstricter\sinterpretation\sof\sJSON\sand\sPATH\narguments. +D 2015-08-29T19:41:45.279 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 5f39a87bf0697cca51e05a83cc5597a431d504de +F ext/misc/json1.c bd51e8c1e8ce580e6f21493bd8e94ed5aca3d777 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P daff4832af963f98bcc1d2c2f84bd815d384f850 -R 082d6d0f70d8c030232efb6a589595f2 +P 2e8e239cec5a12ac81cf62c0fbe94fb5713c31b1 +R e2bf483acb1667e13821f8d776f13acb U drh -Z 988619e0978b399609ee7513adf0867a +Z ec66207700973851a4a5019af1bd75d1 diff --git a/manifest.uuid b/manifest.uuid index d0eeda427c..17d2b0dd8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e8e239cec5a12ac81cf62c0fbe94fb5713c31b1 \ No newline at end of file +1da60c3dda4254620052a83c853c2d2b6dd5009f \ No newline at end of file