diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 5d05da875e..8bbafbaaf4 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -154,6 +154,10 @@ typedef struct Fts5Config Fts5Config; ** attempt to merge together. A value of 1 sets the object to use the ** compile time default. Zero disables auto-merge altogether. ** +** bContentlessDelete: +** True if the contentless_delete option was present in the CREATE +** VIRTUAL TABLE statement. +** ** zContent: ** ** zContentRowid: @@ -188,6 +192,7 @@ struct Fts5Config { int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ int eContent; /* An FTS5_CONTENT value */ + int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */ char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ @@ -209,6 +214,7 @@ struct Fts5Config { char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ int bSecureDelete; /* 'secure-delete' */ + int nDeleteMerge; /* 'deletemerge' */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; @@ -531,6 +537,9 @@ int sqlite3Fts5IndexReset(Fts5Index *p); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); +int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin); +int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ @@ -615,6 +624,11 @@ int sqlite3Fts5HashWrite( */ void sqlite3Fts5HashClear(Fts5Hash*); +/* +** Return true if the hash is empty, false otherwise. +*/ +int sqlite3Fts5HashIsEmpty(Fts5Hash*); + int sqlite3Fts5HashQuery( Fts5Hash*, /* Hash table to query */ int nPre, @@ -636,6 +650,7 @@ void sqlite3Fts5HashScanEntry(Fts5Hash *, ); + /* ** End of interface to code in fts5_hash.c. **************************************************************************/ diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 7a4c7b8177..5d0770502e 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -22,6 +22,8 @@ #define FTS5_DEFAULT_CRISISMERGE 16 #define FTS5_DEFAULT_HASHSIZE (1024*1024) +#define FTS5_DEFAULT_DELETE_AUTOMERGE 10 /* default 10% */ + /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (64*1024) @@ -352,6 +354,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bContentlessDelete = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ if( pConfig->zContentRowid ){ *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); @@ -596,6 +608,28 @@ int sqlite3Fts5ConfigParse( sqlite3_free(zTwo); } + /* We only allow contentless_delete=1 if the table is indeed contentless. */ + if( rc==SQLITE_OK + && pRet->bContentlessDelete + && pRet->eContent!=FTS5_CONTENT_NONE + ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 requires a contentless table" + ); + rc = SQLITE_ERROR; + } + + /* We only allow contentless_delete=1 if columnsize=0 is not present. + ** + ** This restriction may be removed at some point. + */ + if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 is incompatible with columnsize=0" + ); + rc = SQLITE_ERROR; + } + /* If a tokenizer= option was successfully parsed, the tokenizer has ** already been allocated. Otherwise, allocate an instance of the default ** tokenizer (unicode61) now. */ @@ -890,6 +924,18 @@ int sqlite3Fts5ConfigSetValue( } } + else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){ + int nVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nVal = sqlite3_value_int(pVal); + }else{ + *pbBadkey = 1; + } + if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE; + if( nVal>100 ) nVal = 0; + pConfig->nDeleteMerge = nVal; + } + else if( 0==sqlite3_stricmp(zKey, "rank") ){ const char *zIn = (const char*)sqlite3_value_text(pVal); char *zRank; @@ -938,6 +984,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; + pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE; zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); if( zSql ){ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 0e018420d0..0e07b9246c 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2477,7 +2477,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( return pRet; } -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ sqlite3_int64 nByte = 0; Fts5ExprTerm *p; @@ -2844,14 +2844,14 @@ static void fts5ExprFold( sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics)); } } -#endif /* ifdef SQLITE_TEST */ +#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */ /* ** This is called during initialization to register the fts5_expr() scalar ** UDF with the SQLite handle passed as the only argument. */ int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) struct Fts5ExprFunc { const char *z; void (*x)(sqlite3_context*,int,sqlite3_value**); diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index bc9244fc01..7e50c36608 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -475,7 +475,6 @@ static int fts5HashEntrySort( pList = fts5HashEntryMerge(pList, ap[i]); } - pHash->nEntry = 0; sqlite3_free(ap); *ppSorted = pList; return SQLITE_OK; @@ -529,6 +528,28 @@ int sqlite3Fts5HashScanInit( return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); } +#ifdef SQLITE_DEBUG +static int fts5HashCount(Fts5Hash *pHash){ + int nEntry = 0; + int ii; + for(ii=0; iinSlot; ii++){ + Fts5HashEntry *p = 0; + for(p=pHash->aSlot[ii]; p; p=p->pHashNext){ + nEntry++; + } + } + return nEntry; +} +#endif + +/* +** Return true if the hash table is empty, false otherwise. +*/ +int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){ + assert( pHash->nEntry==fts5HashCount(pHash) ); + return pHash->nEntry==0; +} + void sqlite3Fts5HashScanNext(Fts5Hash *p){ assert( !sqlite3Fts5HashScanEof(p) ); p->pScan = p->pScan->pScanNext; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 347ee56908..e46840c2e8 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -56,6 +56,24 @@ #define FTS5_MAX_LEVEL 64 +/* +** There are two versions of the format used for the structure record: +** +** 1. the legacy format, that may be read by all fts5 versions, and +** +** 2. the V2 format, which is used by contentless_delete=1 databases. +** +** Both begin with a 4-byte "configuration cookie" value. Then, a legacy +** format structure record contains a varint - the number of levels in +** the structure. Whereas a V2 structure record contains the constant +** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a +** varint has to be at least 16256 to begin with "0xFF". And the default +** maximum number of levels is 64. +** +** See below for more on structure record formats. +*/ +#define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01" + /* ** Details: ** @@ -63,7 +81,7 @@ ** ** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); ** -** , contains the following 5 types of records. See the comments surrounding +** , contains the following 6 types of records. See the comments surrounding ** the FTS5_*_ROWID macros below for a description of how %_data rowids are ** assigned to each fo them. ** @@ -71,13 +89,13 @@ ** ** The set of segments that make up an index - the index structure - are ** recorded in a single record within the %_data table. The record consists -** of a single 32-bit configuration cookie value followed by a list of -** SQLite varints. If the FTS table features more than one index (because -** there are one or more prefix indexes), it is guaranteed that all share -** the same cookie value. +** of a single 32-bit configuration cookie value followed by a list of +** SQLite varints. ** -** Immediately following the configuration cookie, the record begins with -** three varints: +** If the structure record is a V2 record, the configuration cookie is +** followed by the following 4 bytes: [0xFF 0x00 0x00 0x01]. +** +** Next, the record continues with three varints: ** ** + number of levels, ** + total number of segments on all levels, @@ -92,6 +110,12 @@ ** + first leaf page number (often 1, always greater than 0) ** + final leaf page number ** +** Then, for V2 structures only: +** +** + lower origin counter value, +** + upper origin counter value, +** + the number of tombstone hash pages. +** ** 2. The Averages Record: ** ** A single record within the %_data table. The data is a list of varints. @@ -207,6 +231,38 @@ ** * A list of delta-encoded varints - the first rowid on each subsequent ** child page. ** +** 6. Tombstone Hash Page +** +** These records are only ever present in contentless_delete=1 tables. +** There are zero or more of these associated with each segment. They +** are used to store the tombstone rowids for rows contained in the +** associated segments. +** +** The set of nHashPg tombstone hash pages associated with a single +** segment together form a single hash table containing tombstone rowids. +** To find the page of the hash on which a key might be stored: +** +** iPg = (rowid % nHashPg) +** +** Then, within page iPg, which has nSlot slots: +** +** iSlot = (rowid / nHashPg) % nSlot +** +** Each tombstone hash page begins with an 8 byte header: +** +** 1-byte: Key-size (the size in bytes of each slot). Either 4 or 8. +** 1-byte: rowid-0-tombstone flag. This flag is only valid on the +** first tombstone hash page for each segment (iPg=0). If set, +** the hash table contains rowid 0. If clear, it does not. +** Rowid 0 is handled specially. +** 2-bytes: unused. +** 4-bytes: Big-endian integer containing number of entries on page. +** +** Following this are nSlot 4 or 8 byte slots (depending on the key-size +** in the first byte of the page header). The number of slots may be +** determined based on the size of the page record and the key-size: +** +** nSlot = (nByte - 8) / key-size */ /* @@ -240,6 +296,7 @@ #define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno) #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno) +#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid+(1<<16), 0, 0, ipg) #ifdef SQLITE_DEBUG int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } @@ -275,6 +332,12 @@ struct Fts5Data { /* ** One object per %_data table. +** +** nContentlessDelete: +** The number of contentless delete operations since the most recent +** call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked +** so that extra auto-merge work can be done by fts5IndexFlush() to +** account for the delete operations. */ struct Fts5Index { Fts5Config *pConfig; /* Virtual table configuration */ @@ -289,6 +352,8 @@ struct Fts5Index { int nPendingData; /* Current bytes of pending data */ i64 iWriteRowid; /* Rowid for current doc being written */ int bDelete; /* Current write is a delete */ + int nContentlessDelete; /* Number of contentless delete ops */ + int nPendingRow; /* Number of INSERT in hash table */ /* Error state. */ int rc; /* Current error code */ @@ -323,11 +388,23 @@ struct Fts5DoclistIter { ** The contents of the "structure" record for each index are represented ** using an Fts5Structure record in memory. Which uses instances of the ** other Fts5StructureXXX types as components. +** +** nOriginCntr: +** This value is set to non-zero for structure records created for +** contentlessdelete=1 tables only. In that case it represents the +** origin value to apply to the next top-level segment created. */ struct Fts5StructureSegment { int iSegid; /* Segment id */ int pgnoFirst; /* First leaf page number in segment */ int pgnoLast; /* Last leaf page number in segment */ + + /* contentlessdelete=1 tables only: */ + u64 iOrigin1; + u64 iOrigin2; + int nPgTombstone; /* Number of tombstone hash table pages */ + u64 nEntryTombstone; /* Number of tombstone entries that "count" */ + u64 nEntry; /* Number of rows in this segment */ }; struct Fts5StructureLevel { int nMerge; /* Number of segments in incr-merge */ @@ -337,6 +414,7 @@ struct Fts5StructureLevel { struct Fts5Structure { int nRef; /* Object reference count */ u64 nWriteCounter; /* Total leaves written to level 0 */ + u64 nOriginCntr; /* Origin value for next top-level segment */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ @@ -425,6 +503,13 @@ struct Fts5CResult { ** ** iTermIdx: ** Index of current term on iTermLeafPgno. +** +** apTombstone/nTombstone: +** These are used for contentless_delete=1 tables only. When the cursor +** is first allocated, the apTombstone[] array is allocated so that it +** is large enough for all tombstones hash pages associated with the +** segment. The pages themselves are loaded lazily from the database as +** they are required. */ struct Fts5SegIter { Fts5StructureSegment *pSeg; /* Segment to iterate through */ @@ -433,6 +518,8 @@ struct Fts5SegIter { Fts5Data *pLeaf; /* Current leaf data */ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ i64 iLeafOffset; /* Byte offset within current leaf */ + Fts5Data **apTombstone; /* Array of tombstone pages */ + int nTombstone; /* Next method */ void (*xNext)(Fts5Index*, Fts5SegIter*, int*); @@ -562,6 +649,60 @@ static u16 fts5GetU16(const u8 *aIn){ return ((u16)aIn[0] << 8) + aIn[1]; } +/* +** The only argument points to a buffer at least 8 bytes in size. This +** function interprets the first 8 bytes of the buffer as a 64-bit big-endian +** unsigned integer and returns the result. +*/ +static u64 fts5GetU64(u8 *a){ + return ((u64)a[0] << 56) + + ((u64)a[1] << 48) + + ((u64)a[2] << 40) + + ((u64)a[3] << 32) + + ((u64)a[4] << 24) + + ((u64)a[5] << 16) + + ((u64)a[6] << 8) + + ((u64)a[7] << 0); +} + +/* +** The only argument points to a buffer at least 4 bytes in size. This +** function interprets the first 4 bytes of the buffer as a 32-bit big-endian +** unsigned integer and returns the result. +*/ +static u32 fts5GetU32(const u8 *a){ + return ((u32)a[0] << 24) + + ((u32)a[1] << 16) + + ((u32)a[2] << 8) + + ((u32)a[3] << 0); +} + +/* +** Write iVal, formated as a 64-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU64(u8 *a, u64 iVal){ + a[0] = ((iVal >> 56) & 0xFF); + a[1] = ((iVal >> 48) & 0xFF); + a[2] = ((iVal >> 40) & 0xFF); + a[3] = ((iVal >> 32) & 0xFF); + a[4] = ((iVal >> 24) & 0xFF); + a[5] = ((iVal >> 16) & 0xFF); + a[6] = ((iVal >> 8) & 0xFF); + a[7] = ((iVal >> 0) & 0xFF); +} + +/* +** Write iVal, formated as a 32-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU32(u8 *a, u32 iVal){ + a[0] = ((iVal >> 24) & 0xFF); + a[1] = ((iVal >> 16) & 0xFF); + a[2] = ((iVal >> 8) & 0xFF); + a[3] = ((iVal >> 0) & 0xFF); +} + /* ** Allocate and return a buffer at least nByte bytes in size. ** @@ -789,10 +930,17 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ /* ** Remove all records associated with segment iSegid. */ -static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ +static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){ + int iSegid = pSeg->iSegid; i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; fts5DataDelete(p, iFirst, iLast); + + if( pSeg->nPgTombstone ){ + i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0); + i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1); + fts5DataDelete(p, iTomb1, iTomb2); + } if( p->pIdxDeleter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( @@ -903,11 +1051,19 @@ static int fts5StructureDecode( int nSegment = 0; sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */ Fts5Structure *pRet = 0; /* Structure object to return */ + int bStructureV2 = 0; /* True for FTS5_STRUCTURE_V2 */ + u64 nOriginCntr = 0; /* Largest origin value seen so far */ /* Grab the cookie value */ if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); i = 4; + /* Check if this is a V2 structure record. Set bStructureV2 if it is. */ + if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){ + i += 4; + bStructureV2 = 1; + } + /* Read the total number of levels and segments from the start of the ** structure record. */ i += fts5GetVarint32(&pData[i], nLevel); @@ -958,6 +1114,14 @@ static int fts5StructureDecode( i += fts5GetVarint32(&pData[i], pSeg->iSegid); i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst); i += fts5GetVarint32(&pData[i], pSeg->pgnoLast); + if( bStructureV2 ){ + i += fts5GetVarint(&pData[i], &pSeg->iOrigin1); + i += fts5GetVarint(&pData[i], &pSeg->iOrigin2); + i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntryTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntry); + nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2); + } if( pSeg->pgnoLastpgnoFirst ){ rc = FTS5_CORRUPT; break; @@ -968,6 +1132,9 @@ static int fts5StructureDecode( } } if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT; + if( bStructureV2 ){ + pRet->nOriginCntr = nOriginCntr+1; + } if( rc!=SQLITE_OK ){ fts5StructureRelease(pRet); @@ -1180,6 +1347,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ Fts5Buffer buf; /* Buffer to serialize record into */ int iLvl; /* Used to iterate through levels */ int iCookie; /* Cookie value to store */ + int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9)); assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); memset(&buf, 0, sizeof(Fts5Buffer)); @@ -1188,9 +1356,12 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ iCookie = p->pConfig->iCookie; if( iCookie<0 ) iCookie = 0; - if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){ + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){ sqlite3Fts5Put32(buf.p, iCookie); buf.n = 4; + if( pStruct->nOriginCntr>0 ){ + fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4); + } fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); @@ -1204,9 +1375,17 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ assert( pLvl->nMerge<=pLvl->nSeg ); for(iSeg=0; iSegnSeg; iSeg++){ - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast); + if( pStruct->nOriginCntr>0 ){ + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry); + } } } @@ -1729,6 +1908,23 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ } } +/* +** Allocate a tombstone hash page array (pIter->apTombstone) for the +** iterator passed as the second argument. If an OOM error occurs, leave +** an error in the Fts5Index object. +*/ +static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ + const int nTomb = pIter->pSeg->nPgTombstone; + if( nTomb>0 ){ + Fts5Data **apTomb = 0; + apTomb = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data)*nTomb); + if( apTomb ){ + pIter->apTombstone = apTomb; + pIter->nTombstone = nTomb; + } + } +} + /* ** Initialize the iterator object pIter to iterate through the entries in ** segment pSeg. The iterator is left pointing to the first entry when @@ -1770,6 +1966,7 @@ static void fts5SegIterInit( pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; fts5SegIterLoadTerm(p, pIter, 0); fts5SegIterLoadNPos(p, pIter); + fts5SegIterAllocTombstone(p, pIter); } } @@ -2471,6 +2668,7 @@ static void fts5SegIterSeekInit( } fts5SegIterSetNext(p, pIter); + fts5SegIterAllocTombstone(p, pIter); /* Either: ** @@ -2551,6 +2749,20 @@ static void fts5SegIterHashInit( fts5SegIterSetNext(p, pIter); } +/* +** Array ap[] contains n elements. Release each of these elements using +** fts5DataRelease(). Then free the array itself using sqlite3_free(). +*/ +static void fts5IndexFreeArray(Fts5Data **ap, int n){ + if( ap ){ + int ii; + for(ii=0; iiterm); fts5DataRelease(pIter->pLeaf); fts5DataRelease(pIter->pNextLeaf); + fts5IndexFreeArray(pIter->apTombstone, pIter->nTombstone); fts5DlidxIterFree(pIter->pDlidx); sqlite3_free(pIter->aRowidOffset); memset(pIter, 0, sizeof(Fts5SegIter)); @@ -2895,6 +3108,84 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){ pIter->iSwitchRowid = pSeg->iRowid; } +/* +** The argument to this macro must be an Fts5Data structure containing a +** tombstone hash page. This macro returns the key-size of the hash-page. +*/ +#define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8) + +#define TOMBSTONE_NSLOT(pPg) \ + ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)) : 1) + +/* +** Query a single tombstone hash table for rowid iRowid. Return true if +** it is found or false otherwise. The tombstone hash table is one of +** nHashTable tables. +*/ +static int fts5IndexTombstoneQuery( + Fts5Data *pHash, /* Hash table page to query */ + int nHashTable, /* Number of pages attached to segment */ + u64 iRowid /* Rowid to query hash for */ +){ + const int szKey = TOMBSTONE_KEYSIZE(pHash); + const int nSlot = TOMBSTONE_NSLOT(pHash); + int iSlot = (iRowid / nHashTable) % nSlot; + int nCollide = nSlot; + + if( iRowid==0 ){ + return pHash->p[1]; + }else if( szKey==4 ){ + u32 *aSlot = (u32*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + }else{ + u64 *aSlot = (u64*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + } + + return 0; +} + +/* +** Return true if the iterator passed as the only argument points +** to an segment entry for which there is a tombstone. Return false +** if there is no tombstone or if the iterator is already at EOF. +*/ +static int fts5MultiIterIsDeleted(Fts5Iter *pIter){ + int iFirst = pIter->aFirst[1].iFirst; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + + if( pSeg->pLeaf && pSeg->nTombstone ){ + /* Figure out which page the rowid might be present on. */ + int iPg = ((u64)pSeg->iRowid) % pSeg->nTombstone; + assert( iPg>=0 ); + + /* If tombstone hash page iPg has not yet been loaded from the + ** database, load it now. */ + if( pSeg->apTombstone[iPg]==0 ){ + pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex, + FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg) + ); + if( pSeg->apTombstone[iPg]==0 ) return 0; + } + + return fts5IndexTombstoneQuery( + pSeg->apTombstone[iPg], + pSeg->nTombstone, + pSeg->iRowid + ); + } + + return 0; +} + /* ** Move the iterator to the next entry. ** @@ -2932,7 +3223,9 @@ static void fts5MultiIterNext( fts5AssertMultiIterSetup(p, pIter); assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); - if( pIter->bSkipEmpty==0 || pSeg->nPos ){ + if( (pIter->bSkipEmpty==0 || pSeg->nPos) + && 0==fts5MultiIterIsDeleted(pIter) + ){ pIter->xSetOutputs(pIter, pSeg); return; } @@ -2964,7 +3257,9 @@ static void fts5MultiIterNext2( } fts5AssertMultiIterSetup(p, pIter); - }while( fts5MultiIterIsEmpty(p, pIter) ); + }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter)) + && (p->rc==SQLITE_OK) + ); } } @@ -3519,7 +3814,9 @@ static void fts5MultiIterNew( fts5MultiIterSetEof(pNew); fts5AssertMultiIterSetup(p, pNew); - if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ + if( (pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew)) + || fts5MultiIterIsDeleted(pNew) + ){ fts5MultiIterNext(p, pNew, 0, 0); }else if( pNew->base.bEof==0 ){ Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; @@ -3697,7 +3994,9 @@ static void fts5IndexDiscardData(Fts5Index *p){ if( p->pHash ){ sqlite3Fts5HashClear(p->pHash); p->nPendingData = 0; + p->nPendingRow = 0; } + p->nContentlessDelete = 0; } /* @@ -4334,6 +4633,12 @@ static void fts5IndexMergeLevel( /* Read input from all segments in the input level */ nInput = pLvl->nSeg; + + /* Set the range of origins that will go into the output segment. */ + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1; + pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2; + } } bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); @@ -4393,8 +4698,11 @@ static void fts5IndexMergeLevel( int i; /* Remove the redundant segments from the %_data table */ + assert( pSeg->nEntry==0 ); for(i=0; iaSeg[i].iSegid); + Fts5StructureSegment *pOld = &pLvl->aSeg[i]; + pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone); + fts5DataRemoveSegment(p, pOld); } /* Remove the redundant segments from the input level */ @@ -4420,6 +4728,43 @@ static void fts5IndexMergeLevel( if( pnRem ) *pnRem -= writer.nLeafWritten; } +/* +** If this is not a contentless_delete=1 table, or if the 'deletemerge' +** configuration option is set to 0, then this function always returns -1. +** Otherwise, it searches the structure object passed as the second argument +** for a level suitable for merging due to having a large number of +** tombstones in the tombstone hash. If one is found, its index is returned. +** Otherwise, if there is no suitable level, -1. +*/ +static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){ + Fts5Config *pConfig = p->pConfig; + int iRet = -1; + if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){ + int ii; + int nBest = 0; + + for(ii=0; iinLevel; ii++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[ii]; + i64 nEntry = 0; + i64 nTomb = 0; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + nEntry += pLvl->aSeg[iSeg].nEntry; + nTomb += pLvl->aSeg[iSeg].nEntryTombstone; + } + assert( nEntry>0 || pLvl->nSeg==0 ); + if( nEntry>0 ){ + int nPercent = (nTomb * 100) / nEntry; + if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){ + iRet = ii; + nBest = nPercent; + } + } + } + } + return iRet; +} + /* ** Do up to nPg pages of automerge work on the index. ** @@ -4439,14 +4784,15 @@ static int fts5IndexMerge( int iBestLvl = 0; /* Level offering the most input segments */ int nBest = 0; /* Number of input segments on best level */ - /* Set iBestLvl to the level to read input segments from. */ + /* Set iBestLvl to the level to read input segments from. Or to -1 if + ** there is no level suitable to merge segments from. */ assert( pStruct->nLevel>0 ); for(iLvl=0; iLvlnLevel; iLvl++){ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; if( pLvl->nMerge ){ if( pLvl->nMerge>nBest ){ iBestLvl = iLvl; - nBest = pLvl->nMerge; + nBest = nMin; } break; } @@ -4455,22 +4801,18 @@ static int fts5IndexMerge( iBestLvl = iLvl; } } - - /* If nBest is still 0, then the index must be empty. */ -#ifdef SQLITE_DEBUG - for(iLvl=0; nBest==0 && iLvlnLevel; iLvl++){ - assert( pStruct->aLevel[iLvl].nSeg==0 ); + if( nBestaLevel[iBestLvl].nMerge==0 ){ - break; - } + if( iBestLvl<0 ) break; bRet = 1; fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ fts5StructurePromote(p, iBestLvl+1, pStruct); } + + if( nMin==1 ) nMin = 2; } *ppStruct = pStruct; return bRet; @@ -4983,187 +5325,196 @@ static void fts5FlushOneHash(Fts5Index *p){ /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); - iSegid = fts5AllocateSegid(p, pStruct); fts5StructureInvalidate(p); - if( iSegid ){ - const int pgsz = p->pConfig->pgsz; - int eDetail = p->pConfig->eDetail; - int bSecureDelete = p->pConfig->bSecureDelete; - Fts5StructureSegment *pSeg; /* New segment within pStruct */ - Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ - Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ - - Fts5SegWriter writer; - fts5WriteInit(p, &writer, iSegid); - - pBuf = &writer.writer.buf; - pPgidx = &writer.writer.pgidx; - - /* fts5WriteInit() should have initialized the buffers to (most likely) - ** the maximum space required. */ - assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); - assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); - - /* Begin scanning through hash table entries. This loop runs once for each - ** term/doclist currently stored within the hash table. */ - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); - } - while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ - const char *zTerm; /* Buffer containing term */ - int nTerm; /* Size of zTerm in bytes */ - const u8 *pDoclist; /* Pointer to doclist for this term */ - int nDoclist; /* Size of doclist in bytes */ - - /* Get the term and doclist for this entry. */ - sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - nTerm = (int)strlen(zTerm); - if( bSecureDelete==0 ){ - fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); - if( p->rc!=SQLITE_OK ) break; - assert( writer.bFirstRowidInPage==0 ); + if( sqlite3Fts5HashIsEmpty(pHash)==0 ){ + iSegid = fts5AllocateSegid(p, pStruct); + if( iSegid ){ + const int pgsz = p->pConfig->pgsz; + int eDetail = p->pConfig->eDetail; + int bSecureDelete = p->pConfig->bSecureDelete; + Fts5StructureSegment *pSeg; /* New segment within pStruct */ + Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ + + Fts5SegWriter writer; + fts5WriteInit(p, &writer, iSegid); + + pBuf = &writer.writer.buf; + pPgidx = &writer.writer.pgidx; + + /* fts5WriteInit() should have initialized the buffers to (most likely) + ** the maximum space required. */ + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + + /* Begin scanning through hash table entries. This loop runs once for each + ** term/doclist currently stored within the hash table. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); } - - if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ - /* The entire doclist will fit on the current leaf. */ - fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); - }else{ - int bTermWritten = !bSecureDelete; - i64 iRowid = 0; - i64 iPrev = 0; - int iOff = 0; - - /* The entire doclist will not fit on this leaf. The following - ** loop iterates through the poslists that make up the current - ** doclist. */ - while( p->rc==SQLITE_OK && iOffrc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ + const char *zTerm; /* Buffer containing term */ + int nTerm; /* Size of zTerm in bytes */ + const u8 *pDoclist; /* Pointer to doclist for this term */ + int nDoclist; /* Size of doclist in bytes */ + + /* Get the term and doclist for this entry. */ + sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); + nTerm = (int)strlen(zTerm); + if( bSecureDelete==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + assert( writer.bFirstRowidInPage==0 ); + } + + if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + /* The entire doclist will fit on the current leaf. */ + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); + }else{ + int bTermWritten = !bSecureDelete; + i64 iRowid = 0; + i64 iPrev = 0; + int iOff = 0; + + /* The entire doclist will not fit on this leaf. The following + ** loop iterates through the poslists that make up the current + ** doclist. */ + while( p->rc==SQLITE_OK && iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ iOff++; - nDoclist = 0; - }else{ continue; } } - }else if( (pDoclist[iOff] & 0x01) ){ - fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); - if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ - iOff++; - continue; - } } - } - - if( p->rc==SQLITE_OK && bTermWritten==0 ){ - fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); - bTermWritten = 1; - assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); - } - - if( writer.bFirstRowidInPage ){ - fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); - writer.bFirstRowidInPage = 0; - fts5WriteDlidxAppend(p, &writer, iRowid); - }else{ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); - } - if( p->rc!=SQLITE_OK ) break; - assert( pBuf->n<=pBuf->nSpace ); - iPrev = iRowid; - - if( eDetail==FTS5_DETAIL_NONE ){ - if( iOffp[pBuf->n++] = 0; - iOff++; + + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + bTermWritten = 1; + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); + } + + if( writer.bFirstRowidInPage ){ + fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); + writer.bFirstRowidInPage = 0; + fts5WriteDlidxAppend(p, &writer, iRowid); + }else{ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); + } + if( p->rc!=SQLITE_OK ) break; + assert( pBuf->n<=pBuf->nSpace ); + iPrev = iRowid; + + if( eDetail==FTS5_DETAIL_NONE ){ if( iOffp[pBuf->n++] = 0; iOff++; + if( iOffp[pBuf->n++] = 0; + iOff++; + } + } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); } - } - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); - } - }else{ - int bDummy; - int nPos; - int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); - nCopy += nPos; - if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ - /* The entire poslist will fit on the current leaf. So copy - ** it in one go. */ - fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); }else{ - /* The entire poslist will not fit on this leaf. So it needs - ** to be broken into sections. The only qualification being - ** that each varint must be stored contiguously. */ - const u8 *pPoslist = &pDoclist[iOff]; - int iPos = 0; - while( p->rc==SQLITE_OK ){ - int nSpace = pgsz - pBuf->n - pPgidx->n; - int n = 0; - if( (nCopy - iPos)<=nSpace ){ - n = nCopy - iPos; - }else{ - n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + int bDummy; + int nPos; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); + nCopy += nPos; + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; } - assert( n>0 ); - fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); - iPos += n; - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); - } - if( iPos>=nCopy ) break; } + iOff += nCopy; } - iOff += nCopy; } } + + /* TODO2: Doclist terminator written here. */ + /* pBuf->p[pBuf->n++] = '\0'; */ + assert( pBuf->n<=pBuf->nSpace ); + if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); } - - /* TODO2: Doclist terminator written here. */ - /* pBuf->p[pBuf->n++] = '\0'; */ - assert( pBuf->n<=pBuf->nSpace ); - if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); - } - sqlite3Fts5HashClear(pHash); - fts5WriteFinish(p, &writer, &pgnoLast); - - assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); - if( pgnoLast>0 ){ - /* Update the Fts5Structure. It is written back to the database by the - ** fts5StructureRelease() call below. */ - if( pStruct->nLevel==0 ){ - fts5StructureAddLevel(&p->rc, &pStruct); + sqlite3Fts5HashClear(pHash); + fts5WriteFinish(p, &writer, &pgnoLast); + + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); + if( pgnoLast>0 ){ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pStruct->nOriginCntr; + pSeg->iOrigin2 = pStruct->nOriginCntr; + pSeg->nEntry = p->nPendingRow; + pStruct->nOriginCntr++; + } + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); } - fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); - if( p->rc==SQLITE_OK ){ - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; - pSeg->iSegid = iSegid; - pSeg->pgnoFirst = 1; - pSeg->pgnoLast = pgnoLast; - pStruct->nSegment++; - } - fts5StructurePromote(p, 0, pStruct); } } - fts5IndexAutomerge(p, &pStruct, pgnoLast); + fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete); fts5IndexCrisismerge(p, &pStruct); fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); + p->nContentlessDelete = 0; } /* @@ -5171,10 +5522,11 @@ static void fts5FlushOneHash(Fts5Index *p){ */ static void fts5IndexFlush(Fts5Index *p){ /* Unless it is empty, flush the hash table to disk */ - if( p->nPendingData ){ + if( p->nPendingData || p->nContentlessDelete ){ assert( p->pHash ); - p->nPendingData = 0; fts5FlushOneHash(p); + p->nPendingData = 0; + p->nPendingRow = 0; } } @@ -5190,17 +5542,22 @@ static Fts5Structure *fts5IndexOptimizeStruct( /* Figure out if this structure requires optimization. A structure does ** not require optimization if either: ** - ** + it consists of fewer than two segments, or - ** + all segments are on the same level, or - ** + all segments except one are currently inputs to a merge operation. + ** 1. it consists of fewer than two segments, or + ** 2. all segments are on the same level, or + ** 3. all segments except one are currently inputs to a merge operation. ** - ** In the first case, return NULL. In the second, increment the ref-count - ** on *pStruct and return a copy of the pointer to it. + ** In the first case, if there are no tombstone hash pages, return NULL. In + ** the second, increment the ref-count on *pStruct and return a copy of the + ** pointer to it. */ - if( nSeg<2 ) return 0; + if( nSeg==0 ) return 0; for(i=0; inLevel; i++){ int nThis = pStruct->aLevel[i].nSeg; - if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ + int nMerge = pStruct->aLevel[i].nMerge; + if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){ + if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){ + return 0; + } fts5StructureRef(pStruct); return pStruct; } @@ -5216,6 +5573,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL); pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; + pNew->nOriginCntr = pStruct->nOriginCntr; pLvl = &pNew->aLevel[pNew->nLevel-1]; pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pLvl->aSeg ){ @@ -5246,6 +5604,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); + assert( p->nContentlessDelete==0 ); pStruct = fts5StructureRead(p); fts5StructureInvalidate(p); @@ -5275,7 +5634,10 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ ** INSERT command. */ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ - Fts5Structure *pStruct = fts5StructureRead(p); + Fts5Structure *pStruct = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); if( pStruct ){ int nMin = p->pConfig->nUsermerge; fts5StructureInvalidate(p); @@ -5283,7 +5645,7 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); fts5StructureRelease(pStruct); pStruct = pNew; - nMin = 2; + nMin = 1; nMerge = nMerge*-1; } if( pStruct && pStruct->nLevel ){ @@ -5797,6 +6159,9 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ p->iWriteRowid = iRowid; p->bDelete = bDelete; + if( bDelete==0 ){ + p->nPendingRow++; + } return fts5IndexReturn(p); } @@ -5834,6 +6199,9 @@ int sqlite3Fts5IndexReinit(Fts5Index *p){ fts5StructureInvalidate(p); fts5IndexDiscardData(p); memset(&s, 0, sizeof(Fts5Structure)); + if( p->pConfig->bContentlessDelete ){ + s.nOriginCntr = 1; + } fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); return fts5IndexReturn(p); @@ -6225,6 +6593,347 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ return fts5IndexReturn(p); } +/* +** Retrieve the origin value that will be used for the segment currently +** being accumulated in the in-memory hash table when it is flushed to +** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to +** the queried value. Or, if an error occurs, an error code is returned +** and the final value of (*piOrigin) is undefined. +*/ +int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + *piOrigin = pStruct->nOriginCntr; + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} + +/* +** Buffer pPg contains a page of a tombstone hash table - one of nPg pages +** associated with the same segment. This function adds rowid iRowid to +** the hash table. The caller is required to guarantee that there is at +** least one free slot on the page. +** +** If parameter bForce is false and the hash table is deemed to be full +** (more than half of the slots are occupied), then non-zero is returned +** and iRowid not inserted. Or, if bForce is true or if the hash table page +** is not full, iRowid is inserted and zero returned. +*/ +static int fts5IndexTombstoneAddToPage( + Fts5Data *pPg, + int bForce, + int nPg, + u64 iRowid +){ + const int szKey = TOMBSTONE_KEYSIZE(pPg); + const int nSlot = TOMBSTONE_NSLOT(pPg); + const int nElem = fts5GetU32(&pPg->p[4]); + int iSlot = (iRowid / nPg) % nSlot; + int nCollide = nSlot; + + if( szKey==4 && iRowid>0xFFFFFFFF ) return 2; + if( iRowid==0 ){ + pPg->p[1] = 0x01; + return 0; + } + + if( bForce==0 && nElem>=(nSlot/2) ){ + return 1; + } + + fts5PutU32(&pPg->p[4], nElem+1); + if( szKey==4 ){ + u32 *aSlot = (u32*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid); + }else{ + u64 *aSlot = (u64*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU64((u8*)&aSlot[iSlot], iRowid); + } + + return 0; +} + +/* +** This function attempts to build a new hash containing all the keys +** currently in the tombstone hash table for segment pSeg. The new +** hash will be stored in the nOut buffers passed in array apOut[]. +** All pages of the new hash use key-size szKey (4 or 8). +** +** Return 0 if the hash is successfully rebuilt into the nOut pages. +** Or non-zero if it is not (because one page became overfull). In this +** case the caller should retry with a larger nOut parameter. +** +** Parameter pData1 is page iPg1 of the hash table being rebuilt. +*/ +static int fts5IndexTombstoneRehash( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int nOut, /* Number of output pages */ + Fts5Data **apOut /* Array of output hash pages */ +){ + int ii; + int res = 0; + + /* Initialize the headers of all the output pages */ + for(ii=0; iip[0] = szKey; + fts5PutU32(&apOut[ii]->p[4], 0); + } + + /* Loop through the current pages of the hash table. */ + for(ii=0; res==0 && iinPgTombstone; ii++){ + Fts5Data *pData = 0; /* Page ii of the current hash table */ + Fts5Data *pFree = 0; /* Free this at the end of the loop */ + + if( iPg1==ii ){ + pData = pData1; + }else{ + pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)); + } + + if( pData ){ + int szKeyIn = TOMBSTONE_KEYSIZE(pData); + int nSlotIn = (pData->nn - 8) / szKeyIn; + int iIn; + for(iIn=0; iInp[8]; + if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]); + }else{ + u64 *aSlot = (u64*)&pData->p[8]; + if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]); + } + + /* If iVal is not 0 at this point, insert it into the new hash table */ + if( iVal ){ + Fts5Data *pPg = apOut[(iVal % nOut)]; + res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal); + if( res ) break; + } + } + + /* If this is page 0 of the old hash, copy the rowid-0-flag from the + ** old hash to the new. */ + if( ii==0 ){ + apOut[0]->p[1] = pData->p[1]; + } + } + fts5DataRelease(pFree); + } + + return res; +} + +/* +** This is called to rebuild the hash table belonging to segment pSeg. +** If parameter pData1 is not NULL, then one page of the existing hash table +** has already been loaded - pData1, which is page iPg1. The key-size for +** the new hash table is szKey (4 or 8). +** +** If successful, the new hash table is not written to disk. Instead, +** output parameter (*pnOut) is set to the number of pages in the new +** hash table, and (*papOut) to point to an array of buffers containing +** the new page data. +** +** If an error occurs, an error code is left in the Fts5Index object and +** both output parameters set to 0 before returning. +*/ +static void fts5IndexTombstoneRebuild( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int *pnOut, /* OUT: Number of output pages */ + Fts5Data ***papOut /* OUT: Output hash pages */ +){ + const int MINSLOT = 32; + int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey); + int nSlot = 0; /* Number of slots in each output page */ + int nOut = 0; + + /* Figure out how many output pages (nOut) and how many slots per + ** page (nSlot). There are three possibilities: + ** + ** 1. The hash table does not yet exist. In this case the new hash + ** table will consist of a single page with MINSLOT slots. + ** + ** 2. The hash table exists but is currently a single page. In this + ** case an attempt is made to grow the page to accommodate the new + ** entry. The page is allowed to grow up to nSlotPerPage (see above) + ** slots. + ** + ** 3. The hash table already consists of more than one page, or of + ** a single page already so large that it cannot be grown. In this + ** case the new hash consists of (nPg*2+1) pages of nSlotPerPage + ** slots each, where nPg is the current number of pages in the + ** hash table. + */ + if( pSeg->nPgTombstone==0 ){ + /* Case 1. */ + nOut = 1; + nSlot = MINSLOT; + }else if( pSeg->nPgTombstone==1 ){ + /* Case 2. */ + int nElem = (int)fts5GetU32(&pData1->p[4]); + assert( pData1 && iPg1==0 ); + nOut = 1; + nSlot = MAX(nElem*4, MINSLOT); + if( nSlot>nSlotPerPage ) nOut = 0; + } + if( nOut==0 ){ + /* Case 3. */ + nOut = (pSeg->nPgTombstone * 2 + 1); + nSlot = nSlotPerPage; + } + + /* Allocate the required array and output pages */ + while( 1 ){ + int res = 0; + int ii = 0; + int szPage = 0; + Fts5Data **apOut = 0; + + /* Allocate space for the new hash table */ + assert( nSlot>=MINSLOT ); + apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut); + szPage = 8 + nSlot*szKey; + for(ii=0; iirc, + sizeof(Fts5Data)+szPage + ); + if( pNew ){ + pNew->nn = szPage; + pNew->p = (u8*)&pNew[1]; + apOut[ii] = pNew; + } + } + + /* Rebuild the hash table. */ + if( p->rc==SQLITE_OK ){ + res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut); + } + if( res==0 ){ + if( p->rc ){ + fts5IndexFreeArray(apOut, nOut); + apOut = 0; + nOut = 0; + } + *pnOut = nOut; + *papOut = apOut; + break; + } + + /* If control flows to here, it was not possible to rebuild the hash + ** table. Free all buffers and then try again with more pages. */ + assert( p->rc==SQLITE_OK ); + fts5IndexFreeArray(apOut, nOut); + nSlot = nSlotPerPage; + nOut = nOut*2 + 1; + } +} + + +/* +** Add a tombstone for rowid iRowid to segment pSeg. +*/ +static void fts5IndexTombstoneAdd( + Fts5Index *p, + Fts5StructureSegment *pSeg, + u64 iRowid +){ + Fts5Data *pPg = 0; + int iPg = -1; + int szKey = 0; + int nHash = 0; + Fts5Data **apHash = 0; + + p->nContentlessDelete++; + + if( pSeg->nPgTombstone>0 ){ + iPg = iRowid % pSeg->nPgTombstone; + pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)); + if( pPg==0 ){ + assert( p->rc!=SQLITE_OK ); + return; + } + + if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){ + fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn); + fts5DataRelease(pPg); + return; + } + } + + /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */ + szKey = pPg ? TOMBSTONE_KEYSIZE(pPg) : 4; + if( iRowid>0xFFFFFFFF ) szKey = 8; + + /* Rebuild the hash table */ + fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash); + assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) ); + + /* If all has succeeded, write the new rowid into one of the new hash + ** table pages, then write them all out to disk. */ + if( nHash ){ + int ii = 0; + fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid); + for(ii=0; iiiSegid, ii); + fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn); + } + pSeg->nPgTombstone = nHash; + fts5StructureWrite(p, p->pStruct); + } + + fts5DataRelease(pPg); + fts5IndexFreeArray(apHash, nHash); +} + +/* +** Add iRowid to the tombstone list of the segment or segments that contain +** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite +** error code otherwise. +*/ +int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + int bFound = 0; /* True after pSeg->nEntryTombstone incr. */ + int iLvl; + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ + int iSeg; + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){ + if( bFound==0 ){ + pSeg->nEntryTombstone++; + bFound = 1; + } + fts5IndexTombstoneAdd(p, pSeg, iRowid); + } + } + } + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} /************************************************************************* ************************************************************************** @@ -6776,13 +7485,14 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ ** function only. */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** Decode a segment-data rowid from the %_data table. This function is ** the opposite of macro FTS5_SEGMENT_ROWID(). */ static void fts5DecodeRowid( i64 iRowid, /* Rowid from %_data table */ + int *pbTombstone, /* OUT: Tombstone hash flag */ int *piSegid, /* OUT: Segment id */ int *pbDlidx, /* OUT: Dlidx flag */ int *piHeight, /* OUT: Height */ @@ -6798,13 +7508,16 @@ static void fts5DecodeRowid( iRowid >>= FTS5_DATA_DLI_B; *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); -} -#endif /* SQLITE_TEST */ + iRowid >>= FTS5_DATA_ID_B; -#ifdef SQLITE_TEST + *pbTombstone = (int)(iRowid & 0x0001); +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ - int iSegid, iHeight, iPgno, bDlidx; /* Rowid compenents */ - fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno); + int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid compenents */ + fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); if( iSegid==0 ){ if( iKey==FTS5_AVERAGES_ROWID ){ @@ -6814,14 +7527,16 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ } } else{ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", - bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}", + bDlidx ? "dlidx " : "", + bTomb ? "tombstone " : "", + iSegid, iHeight, iPgno ); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugStructure( int *pRc, /* IN/OUT: error code */ Fts5Buffer *pBuf, @@ -6836,16 +7551,22 @@ static void fts5DebugStructure( ); for(iSeg=0; iSegnSeg; iSeg++){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d", pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast ); + if( pSeg->iOrigin1>0 ){ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld", + pSeg->iOrigin1, pSeg->iOrigin2 + ); + } + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This is part of the fts5_decode() debugging aid. ** @@ -6870,9 +7591,9 @@ static void fts5DecodeStructure( fts5DebugStructure(pRc, pBuf, p); fts5StructureRelease(p); } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This is part of the fts5_decode() debugging aid. ** @@ -6895,9 +7616,9 @@ static void fts5DecodeAverages( zSpace = " "; } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** Buffer (a/n) is assumed to contain a list of serialized varints. Read ** each varint and append its string representation to buffer pBuf. Return @@ -6914,9 +7635,9 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ } return iOff; } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The start of buffer (a/n) contains the start of a doclist. The doclist ** may or may not finish within the buffer. This function appends a text @@ -6949,9 +7670,9 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ return iOff; } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This function is part of the fts5_decode() debugging function. It is ** only ever used with detail=none tables. @@ -6992,9 +7713,9 @@ static void fts5DecodeRowidList( sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The implementation of user-defined scalar function fts5_decode(). */ @@ -7005,6 +7726,7 @@ static void fts5DecodeFunction( ){ i64 iRowid; /* Rowid for record being decoded */ int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ + int bTomb; const u8 *aBlob; int n; /* Record to decode */ u8 *a = 0; Fts5Buffer s; /* Build up text to return here */ @@ -7027,7 +7749,7 @@ static void fts5DecodeFunction( if( a==0 ) goto decode_out; if( n>0 ) memcpy(a, aBlob, n); - fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno); + fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); fts5DebugRowid(&rc, &s, iRowid); if( bDlidx ){ @@ -7046,6 +7768,28 @@ static void fts5DecodeFunction( " %d(%lld)", lvl.iLeafPgno, lvl.iRowid ); } + }else if( bTomb ){ + u32 nElem = fts5GetU32(&a[4]); + int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8; + int nSlot = (n - 8) / szKey; + int ii; + sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem); + if( aBlob[1] ){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0"); + } + for(ii=0; iiestimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 0; + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->usable==0 ) continue; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==11 ){ + rc = SQLITE_OK; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + break; + } + } + return rc; +} + +/* +** This method is the destructor for bytecodevtab objects. +*/ +static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5StructVtab *p = (Fts5StructVtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new bytecodevtab_cursor object. +*/ +static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ + int rc = SQLITE_OK; + Fts5StructVcsr *pNew = 0; + + pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + *ppCsr = (sqlite3_vtab_cursor*)pNew; + + return SQLITE_OK; +} + +/* +** Destructor for a bytecodevtab_cursor. +*/ +static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + fts5StructureRelease(pCsr->pStruct); + sqlite3_free(pCsr); + return SQLITE_OK; +} + + +/* +** Advance a bytecodevtab_cursor to its next row of output. +*/ +static int fts5structNextMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + + assert( pCsr->pStruct ); + pCsr->iSeg++; + pCsr->iRowid++; + while( pCsr->iSeg>=pCsr->pStruct->aLevel[pCsr->iLevel].nSeg ){ + pCsr->iLevel++; + pCsr->iSeg = 0; + } + if( pCsr->iLevel>=pCsr->pStruct->nLevel ){ + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + } + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fts5structEofMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + return pCsr->pStruct==0; +} + +static int fts5structRowidMethod( + sqlite3_vtab_cursor *cur, + sqlite_int64 *piRowid +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + *piRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the bytecodevtab_cursor +** is currently pointing. +*/ +static int fts5structColumnMethod( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg]; + + switch( i ){ + case 0: /* level */ + sqlite3_result_int(ctx, pCsr->iLevel); + break; + case 1: /* segment */ + sqlite3_result_int(ctx, pCsr->iSeg); + break; + case 2: /* merge */ + sqlite3_result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge); + break; + case 3: /* segid */ + sqlite3_result_int(ctx, pSeg->iSegid); + break; + case 4: /* leaf1 */ + sqlite3_result_int(ctx, pSeg->pgnoFirst); + break; + case 5: /* leaf2 */ + sqlite3_result_int(ctx, pSeg->pgnoLast); + break; + case 6: /* origin1 */ + sqlite3_result_int64(ctx, pSeg->iOrigin1); + break; + case 7: /* origin2 */ + sqlite3_result_int64(ctx, pSeg->iOrigin2); + break; + case 8: /* npgtombstone */ + sqlite3_result_int(ctx, pSeg->nPgTombstone); + break; + case 9: /* nentrytombstone */ + sqlite3_result_int64(ctx, pSeg->nEntryTombstone); + break; + case 10: /* nentry */ + sqlite3_result_int64(ctx, pSeg->nEntry); + break; + } + return SQLITE_OK; +} + +/* +** Initialize a cursor. +** +** idxNum==0 means show all subprograms +** idxNum==1 means show only the main bytecode and omit subprograms. +*/ +static int fts5structFilterMethod( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor; + int rc = SQLITE_OK; + + const u8 *aBlob = 0; + int nBlob = 0; + + assert( argc==1 ); + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + + nBlob = sqlite3_value_bytes(argv[0]); + aBlob = (const u8*)sqlite3_value_blob(argv[0]); + rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct); + if( rc==SQLITE_OK ){ + pCsr->iLevel = 0; + pCsr->iRowid = 0; + pCsr->iSeg = -1; + rc = fts5structNextMethod(pVtabCursor); + } + + return rc; +} + +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ /* ** This is called as part of registering the FTS5 module with database @@ -7250,7 +8221,7 @@ static void fts5RowidFunction( ** SQLite error code is returned instead. */ int sqlite3Fts5IndexInit(sqlite3 *db){ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) int rc = sqlite3_create_function( db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 ); @@ -7267,6 +8238,36 @@ int sqlite3Fts5IndexInit(sqlite3 *db){ db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 ); } + + if( rc==SQLITE_OK ){ + static const sqlite3_module fts5structure_module = { + 0, /* iVersion */ + 0, /* xCreate */ + fts5structConnectMethod, /* xConnect */ + fts5structBestIndexMethod, /* xBestIndex */ + fts5structDisconnectMethod, /* xDisconnect */ + 0, /* xDestroy */ + fts5structOpenMethod, /* xOpen */ + fts5structCloseMethod, /* xClose */ + fts5structFilterMethod, /* xFilter */ + fts5structNextMethod, /* xNext */ + fts5structEofMethod, /* xEof */ + fts5structColumnMethod, /* xColumn */ + fts5structRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0); + } return rc; #else return SQLITE_OK; diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 13921ce49e..d1c79d3604 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1624,7 +1624,6 @@ static int fts5UpdateMethod( int eType0; /* value_type() of apVal[0] */ int rc = SQLITE_OK; /* Return code */ int bUpdateOrDelete = 0; - /* A transaction must be open when this is called. */ assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); @@ -1654,7 +1653,14 @@ static int fts5UpdateMethod( if( pConfig->eContent!=FTS5_CONTENT_NORMAL && 0==sqlite3_stricmp("delete", z) ){ - rc = fts5SpecialDelete(pTab, apVal); + if( pConfig->bContentlessDelete ){ + fts5SetVtabError(pTab, + "'delete' may not be used with a contentless_delete=1 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = fts5SpecialDelete(pTab, apVal); + } }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); } @@ -1671,7 +1677,7 @@ static int fts5UpdateMethod( ** Cases 3 and 4 may violate the rowid constraint. */ int eConflict = SQLITE_ABORT; - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){ eConflict = sqlite3_vtab_on_conflict(pConfig->db); } @@ -1679,8 +1685,12 @@ static int fts5UpdateMethod( assert( nArg!=1 || eType0==SQLITE_INTEGER ); /* Filter out attempts to run UPDATE or DELETE on contentless tables. - ** This is not suported. */ - if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){ + ** This is not suported. Except - DELETE is supported if the CREATE + ** VIRTUAL TABLE statement contained "contentless_delete=1". */ + if( eType0==SQLITE_INTEGER + && pConfig->eContent==FTS5_CONTENT_NONE + && (nArg>1 || pConfig->bContentlessDelete==0) + ){ pTab->p.base.zErrMsg = sqlite3_mprintf( "cannot %s contentless fts5 table: %s", (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName @@ -1767,8 +1777,7 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; fts5CheckTransactionState(pTab, FTS5_SYNC, 0); pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; - fts5TripCursors(pTab); - rc = sqlite3Fts5StorageSync(pTab->pStorage); + rc = sqlite3Fts5FlushToDisk(&pTab->p); pTab->p.pConfig->pzErrmsg = 0; return rc; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 02b98d9e44..0a0af9d4b5 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -77,10 +77,10 @@ static int fts5StorageGetStmt( "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ - "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */ + "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */ "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ - "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ + "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ "SELECT %s FROM %s AS T", /* SCAN */ @@ -128,6 +128,19 @@ static int fts5StorageGetStmt( break; } + case FTS5_STMT_REPLACE_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, + (pC->bContentlessDelete ? ",?" : "") + ); + break; + + case FTS5_STMT_LOOKUP_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], + (pC->bContentlessDelete ? ",origin" : ""), + pC->zDb, pC->zName + ); + break; + default: zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); break; @@ -317,9 +330,11 @@ int sqlite3Fts5StorageOpen( } if( rc==SQLITE_OK && pConfig->bColumnsize ){ - rc = sqlite3Fts5CreateTable( - pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr - ); + const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB"; + if( pConfig->bContentlessDelete ){ + zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER"; + } + rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable( @@ -396,7 +411,7 @@ static int fts5StorageDeleteFromIndex( ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ int rc2; /* sqlite3_reset() return code */ int iCol; Fts5InsertCtx ctx; @@ -412,7 +427,6 @@ static int fts5StorageDeleteFromIndex( ctx.pStorage = p; ctx.iCol = -1; - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1]==0 ){ const char *zText; @@ -449,6 +463,37 @@ static int fts5StorageDeleteFromIndex( return rc; } +/* +** This function is called to process a DELETE on a contentless_delete=1 +** table. It adds the tombstone required to delete the entry with rowid +** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs, +** an SQLite error code. +*/ +static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){ + i64 iOrigin = 0; + sqlite3_stmt *pLookup = 0; + int rc = SQLITE_OK; + + assert( p->pConfig->bContentlessDelete ); + assert( p->pConfig->eContent==FTS5_CONTENT_NONE ); + + /* Look up the origin of the document in the %_docsize table. Store + ** this in stack variable iOrigin. */ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLookup, 1, iDel); + if( SQLITE_ROW==sqlite3_step(pLookup) ){ + iOrigin = sqlite3_column_int64(pLookup, 1); + } + rc = sqlite3_reset(pLookup); + } + + if( rc==SQLITE_OK && iOrigin!=0 ){ + rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel); + } + + return rc; +} /* ** Insert a record into the %_docsize table. Specifically, do: @@ -469,10 +514,17 @@ static int fts5StorageInsertDocsize( rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pReplace, 1, iRowid); - sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); - sqlite3_step(pReplace); - rc = sqlite3_reset(pReplace); - sqlite3_bind_null(pReplace, 2); + if( p->pConfig->bContentlessDelete ){ + i64 iOrigin = 0; + rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin); + sqlite3_bind_int64(pReplace, 3, iOrigin); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); + } } } return rc; @@ -536,7 +588,15 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ /* Delete the index records */ if( rc==SQLITE_OK ){ - rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); + } + + if( rc==SQLITE_OK ){ + if( p->pConfig->bContentlessDelete ){ + rc = fts5StorageContentlessDelete(p, iDel); + }else{ + rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + } } /* Delete the %_docsize record */ diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test new file mode 100644 index 0000000000..41a78e9c8f --- /dev/null +++ b/ext/fts5/test/fts5contentless.test @@ -0,0 +1,271 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# Check that it is not possible to specify "contentless_delete=1" for +# anything other than a contentless table. +# +set res(0) {0 {}} +set res(1) {1 {contentless_delete=1 requires a contentless table}} +foreach {tn sql bError} { + 1 "(a, b, contentless_delete=1)" 1 + 2 "(a, b, contentless_delete=1, content=abc)" 1 + 3 "(a, b, contentless_delete=1, content=)" 0 + 4 "(content=, contentless_delete=1, a)" 0 + 5 "(content='', contentless_delete=1, hello)" 0 +} { + execsql { BEGIN } + do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError) + execsql { ROLLBACK } +} + +# Check that it is not possible to specify "contentless_delete=1" +# along with columnsize=1. +# +set res(0) {0 {}} +set res(1) {1 {contentless_delete=1 is incompatible with columnsize=0}} +foreach {tn sql bError} { + 2 "(a, b, content='', contentless_delete=1, columnsize=0)" 1 +} { + execsql { BEGIN } + do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError) + execsql { ROLLBACK } +} + +# Check that if contentless_delete=1 is specified, then the "origin" +# column is added to the %_docsize table. +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts5(c, content=''); + CREATE VIRTUAL TABLE x2 USING fts5(c, content='', contentless_delete=1); +} +do_execsql_test 3.1 { + SELECT sql FROM sqlite_schema WHERE name IN ('x1_docsize', 'x2_docsize'); +} { + {CREATE TABLE 'x1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} + {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER)} +} + +do_execsql_test 3.2.1 { + SELECT hex(block) FROM x1_data WHERE id=10 +} {00000000000000} +do_execsql_test 3.2.2 { + SELECT hex(block) FROM x2_data WHERE id=10 +} {00000000FF000001000000} + +do_execsql_test 3.3 { + INSERT INTO x2 VALUES('first text'); + INSERT INTO x2 VALUES('second text'); +} +do_execsql_test 3.4 { + SELECT id, origin FROM x2_docsize +} {1 1 2 2} +do_execsql_test 3.5 { + SELECT level, segment, loc1, loc2 FROM fts5_structure( + (SELECT block FROM x2_data WHERE id=10) + ) +} { + 0 0 1 1 + 0 1 2 2 +} +do_execsql_test 3.6 { + INSERT INTO x2(x2) VALUES('optimize'); +} +do_execsql_test 3.7 { + SELECT level, segment, loc1, loc2 FROM fts5_structure( + (SELECT block FROM x2_data WHERE id=10) + ) +} { + 1 0 1 2 +} + +do_execsql_test 3.8 { + DELETE FROM x2 WHERE rowid=2; +} + +do_execsql_test 3.9 { + SELECT rowid FROM x2('text') +} {1} + +#-------------------------------------------------------------------------- +reset_db +proc document {n} { + set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z] + set ret [list] + for {set ii 0} {$ii < $n} {incr ii} { + lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]] + } + set ret +} + +set nRow 1000 + +do_execsql_test 4.0 { + CREATE TABLE t1(x); + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(ft, rank) VALUES('pgsz', 100); +} +do_test 4.1 { + for {set ii 0} {$ii < $nRow} {incr ii} { + set doc [document 6] + execsql { + INSERT INTO t1 VALUES($doc); + INSERT INTO ft VALUES($doc); + } + } +} {} + +foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} { + set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}] + set L2 [execsql {SELECT rowid FROM ft($v)}] + do_test 4.2.$v { set L1 } $L2 +} + +do_test 4.3 { + for {set ii 1} {$ii < $nRow} {incr ii 2} { + execsql { + DELETE FROM ft WHERE rowid=$ii; + DELETE FROM t1 WHERE rowid=$ii; + } + } +} {} + +foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} { + set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}] + set L2 [execsql {SELECT rowid FROM ft($v)}] + do_test 4.4.$v { set L1 } $L2 +} + +do_execsql_test 4.5 { + INSERT INTO ft(ft) VALUES('optimize'); +} {} + +foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} { + set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}] + set L2 [execsql {SELECT rowid FROM ft($v)}] + do_test 4.6.$v { set L1 } $L2 +} + +#execsql_pp { SELECT fts5_decode(id, block) FROM ft_data } + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(rowid, x) VALUES(1, 'one two three'); + INSERT INTO ft(rowid, x) VALUES(2, 'one two four'); + INSERT INTO ft(rowid, x) VALUES(3, 'one two five'); + INSERT INTO ft(rowid, x) VALUES(4, 'one two seven'); + INSERT INTO ft(rowid, x) VALUES(5, 'one two eight'); +} + +do_execsql_test 5.1 { + DELETE FROM ft WHERE rowid=2 +} + +do_execsql_test 5.2 { + SELECT rowid FROM ft +} {1 3 4 5} + +do_catchsql_test 5.3 { + UPDATE ft SET x='four six' WHERE rowid=3 +} {1 {cannot UPDATE contentless fts5 table: ft}} + +do_execsql_test 5.4 { + SELECT rowid FROM ft('one'); +} {1 3 4 5} + +do_execsql_test 5.5 { + REPLACE INTO ft(rowid, x) VALUES(3, 'four six'); + SELECT rowid FROM ft('one'); +} {1 4 5} + +do_execsql_test 5.6 { + REPLACE INTO ft(rowid, x) VALUES(6, 'one two eleven'); + SELECT rowid FROM ft('one'); +} {1 4 5 6} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(rowid, x) VALUES(1, 'one two three'); + INSERT INTO ft(rowid, x) VALUES(2, 'one two four'); +} + +do_test 6.1 { + db eval { SELECT rowid FROM ft('one two') } { + if {$rowid==1} { + db eval { INSERT INTO ft(rowid, x) VALUES(3, 'one two four') } + } + } +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); +} + +set lRowid [list -450 0 1 2 42] + +do_test 7.1 { + execsql BEGIN + foreach r $lRowid { + execsql { INSERT INTO ft(rowid, x) VALUES($r, 'one one one'); } + } + execsql COMMIT +} {} + +do_test 7.2 { + execsql BEGIN + foreach r $lRowid { + execsql { REPLACE INTO ft(rowid, x) VALUES($r, 'two two two'); } + } + execsql COMMIT +} {} + +do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {} +do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft VALUES('hello world'); + INSERT INTO ft VALUES('one two three'); +} + +do_catchsql_test 8.1 { + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world'); +} {1 {'delete' may not be used with a contentless_delete=1 table}} + +do_execsql_test 8.2 { + BEGIN; + INSERT INTO ft(rowid, x) VALUES(3, 'four four four'); + DELETE FROM ft WHERE rowid=3; + COMMIT; + SELECT rowid FROM ft('four'); +} {} + +finish_test + diff --git a/ext/fts5/test/fts5contentless2.test b/ext/fts5/test/fts5contentless2.test new file mode 100644 index 0000000000..fbb857ab38 --- /dev/null +++ b/ext/fts5/test/fts5contentless2.test @@ -0,0 +1,208 @@ +# 2023 July 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless2 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc vocab {} { + list aaa bbb ccc ddd eee fff ggg hhh iii jjj kkk lll mmm nnn ooo ppp +} + +proc document {nToken} { + set doc [list] + set vocab [vocab] + for {set ii 0} {$ii < $nToken} {incr ii} { + lappend doc [lindex $vocab [expr int(rand()*[llength $vocab])]] + } + set doc +} +db func document document + +proc contains {doc token} { + expr {[lsearch $doc $token]>=0} +} +db func contains contains + +proc do_compare_tables_test {tn} { + uplevel [list do_test $tn { + foreach v [vocab] { + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $v) }] + set l2 [execsql { SELECT rowid FROM t2($v) }] + if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" } + + set w "[string range $v 0 1]*" + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }] + set l2 [execsql { SELECT rowid FROM t2($w) }] + if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" } + + set w "[string range $v 0 0]*" + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }] + set l2 [execsql { SELECT rowid FROM t2($w) }] + if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" } + + set l1 [execsql { + SELECT rowid FROM t1 WHERE contains(doc, $v) ORDER BY rowid DESC + }] + set l2 [execsql { SELECT rowid FROM t2($v) ORDER BY rowid DESC }] + if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" } + } + set {} {} + } {}] +} + +proc lshuffle {in} { + set L [list] + set ret [list] + foreach elem $in { lappend L [list [expr rand()] $elem] } + foreach pair [lsort -index 0 $L] { lappend ret [lindex $pair 1] } + set ret +} + +expr srand(0) + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5( + doc, prefix=2, content=, contentless_delete=1 + ); + + CREATE TABLE t1(doc); + CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN + DELETE FROM t2 WHERE rowid = old.rowid; + END; +} + +set SMALLEST64 -9223372036854775808 +set LARGEST64 9223372036854775807 + +foreach {tn r1 r2} { + 1 0 50 + 2 $SMALLEST64 $SMALLEST64+50 + 3 $LARGEST64-50 $LARGEST64 + 4 -50 -1 +} { + set r1 [expr $r1] + set r2 [expr $r2] + + do_test 1.1.$tn { + execsql BEGIN + for {set ii $r1} {$ii <= $r2} {incr ii} { + execsql { INSERT INTO t1(rowid, doc) VALUES ($ii, document(8)); } + } + execsql COMMIT + } {} +} +do_test 1.2 { + db eval { SELECT rowid, doc FROM t1 } { + execsql { INSERT INTO t2(rowid, doc) VALUES($rowid, $doc) } + } +} {} + +foreach {tn rowid} { + 1 $SMALLEST64 + 2 0 + 3 -5 + 4 -30 + 5 $LARGEST64 + 6 $LARGEST64-1 +} { + set rowid [expr $rowid] + do_execsql_test 1.3.$tn.1 { + DELETE FROM t1 WHERE rowid=$rowid + } + do_compare_tables_test 1.3.$tn.2 +} + +set iTest 1 +foreach r [lshuffle [execsql {SELECT rowid FROM t1}]] { + if {($iTest % 50)==0} { + execsql { INSERT INTO t2(t2) VALUES('optimize') } + } + if {($iTest % 5)==0} { + execsql { INSERT INTO t2(t2, rank) VALUES('merge', 5) } + } + do_execsql_test 1.4.$iTest.1($r) { + DELETE FROM t1 WHERE rowid=$r + } + do_compare_tables_test 1.4.$iTest.2 + incr iTest +} + +do_execsql_test 1.5 { + SELECT * FROM t1 +} {} + +#------------------------------------------------------------------------- +reset_db +db func document document + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s; +} + +do_execsql_test 2.1 { + BEGIN; + DELETE FROM t2 WHERE rowid=32; + DELETE FROM t2 WHERE rowid=64; + DELETE FROM t2 WHERE rowid=96; + DELETE FROM t2 WHERE rowid=128; + DELETE FROM t2 WHERE rowid=160; + DELETE FROM t2 WHERE rowid=192; + COMMIT; +} + +do_execsql_test 2.2 { + SELECT * FROM t2('128'); +} {} + +#------------------------------------------------------------------------- + +foreach {tn step} { + 1 3 + 2 7 + 3 15 +} { + set step [expr $step] + + reset_db + db func document document + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1); + INSERT INTO t2(t2, rank) VALUES('pgsz', 100); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s; + } + do_execsql_test 3.$tn.1 { + DELETE FROM t2 WHERE (rowid % $step)==0 + } + do_execsql_test 3.$tn.2 { + SELECT * FROM t2( $step * 5 ) + } {} +} + + + +finish_test + diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test new file mode 100644 index 0000000000..a44311e45a --- /dev/null +++ b/ext/fts5/test/fts5contentless3.test @@ -0,0 +1,196 @@ +# 2023 July 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless3 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1); + BEGIN; + INSERT INTO ft VALUES('one one one'); + INSERT INTO ft VALUES('two two two'); + INSERT INTO ft VALUES('three three three'); + INSERT INTO ft VALUES('four four four'); + INSERT INTO ft VALUES('five five five'); + INSERT INTO ft VALUES('six six six'); + INSERT INTO ft VALUES('seven seven seven'); + INSERT INTO ft VALUES('eight eight eight'); + INSERT INTO ft VALUES('nine nine nine'); + COMMIT; + + DELETE FROM ft WHERE rowid=3; +} + +proc myhex {hex} { binary decode hex $hex } +db func myhex myhex + +do_execsql_test 1.1 { + UPDATE ft_data SET block = + myhex('04000000 00000001' || + '01020304 01020304 01020304 01020304' || + '01020304 01020304 01020304 01020304' + ) + WHERE id = (SELECT max(id) FROM ft_data); +} + +do_execsql_test 1.2 { + DELETE FROM ft WHERE rowid=1 +} + +do_execsql_test 1.3 { + SELECT rowid FROM ft('two'); +} {2} + +do_execsql_test 1.3 { + UPDATE ft_data SET block = + myhex('08000000 00000001' || + '0000000001020304 0000000001020304 0000000001020304 0000000001020304' || + '0000000001020304 0000000001020304 0000000001020304 0000000001020304' + ) + WHERE id = (SELECT max(id) FROM ft_data); +} + +do_execsql_test 1.4 { + SELECT rowid FROM ft('two'); +} {2} + +do_execsql_test 1.5 { + DELETE FROM ft WHERE rowid=4 +} + +do_execsql_test 1.6 { + UPDATE ft_data SET block = myhex('04000000 00000000') + WHERE id = (SELECT max(id) FROM ft_data); +} +do_execsql_test 1.7 { + SELECT rowid FROM ft('two'); +} {2} + +do_execsql_test 1.8 { + UPDATE ft_data SET block = myhex('04000000 00000000') + WHERE id = (SELECT max(id) FROM ft_data); +} +do_execsql_test 1.9 { + DELETE FROM ft WHERE rowid=8 +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1); + INSERT INTO ft VALUES('one one one'); + INSERT INTO ft VALUES('two two two'); + INSERT INTO ft VALUES('three three three'); + INSERT INTO ft VALUES('four four four'); + INSERT INTO ft VALUES('five five five'); + INSERT INTO ft VALUES('six six six'); + INSERT INTO ft VALUES('seven seven seven'); + INSERT INTO ft VALUES('eight eight eight'); + INSERT INTO ft VALUES('nine nine nine'); +} + +do_execsql_test 2.1 { + INSERT INTO ft(ft) VALUES('optimize'); +} +do_execsql_test 2.2 { + SELECT count(*) FROM ft_data +} {3} +do_execsql_test 2.3 { + DELETE FROM ft WHERE rowid=5 +} +do_execsql_test 2.4 { + SELECT count(*) FROM ft_data +} {4} + +# Check that an 'optimize' works (rewrites the index) if there is a single +# segment with one or more tombstone hash pages. +do_execsql_test 2.5 { + INSERT INTO ft(ft) VALUES('optimize'); +} +do_execsql_test 2.6 { + SELECT count(*) FROM ft_data +} {3} + +# Check that an 'optimize' is a no-op if there is a single segment +# and no tombstone hash pages. +do_execsql_test 2.7 { + INSERT INTO ft(ft) VALUES('optimize'); + SELECT rowid FROM ft_data; +} [db eval {SELECT rowid FROM ft_data}] + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1); + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO ft(rowid, x) SELECT i, i||' '||i||' '||i||' '||i FROM s; + INSERT INTO ft(ft) VALUES('optimize'); +} + +do_execsql_test 3.1 { + SELECT count(*) FROM ft_data +} {200} + +do_execsql_test 3.2 { + DELETE FROM ft WHERE (rowid % 50)==0; + SELECT count(*) FROM ft_data; +} {203} + +do_execsql_test 3.3 { + INSERT INTO ft(ft, rank) VALUES('merge', 500); + SELECT rowid FROM ft_data; +} [db eval {SELECT rowid FROM ft_data}] + +do_execsql_test 3.4 { + INSERT INTO ft(ft, rank) VALUES('merge', -1000); + SELECT count(*) FROM ft_data; +} {197} + +do_execsql_test 3.5 { + DELETE FROM ft WHERE (rowid % 50)==1; + SELECT count(*) FROM ft_data; +} {200} + +do_execsql_test 3.6 { + SELECT level, segment, npgtombstone FROM fts5_structure( + (SELECT block FROM ft_data WHERE id=10) + ) +} {1 0 3} + +do_test 3.6 { + while 1 { + set nChange [db total_changes] + execsql { INSERT INTO ft(ft, rank) VALUES('merge', -5) } + if {([db total_changes] - $nChange)<2} break + } +} {} + +do_execsql_test 3.7 { + SELECT level, segment, npgtombstone FROM fts5_structure( + (SELECT block FROM ft_data WHERE id=10) + ) +} {2 0 0} + + +finish_test + diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test new file mode 100644 index 0000000000..1c2666dcf8 --- /dev/null +++ b/ext/fts5/test/fts5contentless4.test @@ -0,0 +1,248 @@ +# 2023 July 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless4 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc document {n} { + set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z] + set ret [list] + for {set ii 0} {$ii < $n} {incr ii} { + lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]] + } + set ret +} +db func document document + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(ft, rank) VALUES('pgsz', 240); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO ft SELECT document(12) FROM s; +} + +do_execsql_test 1.1 { + INSERT INTO ft(ft) VALUES('optimize'); +} + +do_execsql_test 1.2 { + SELECT level, segment, nentry, nentrytombstone FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {0 0 1000 0} + +do_execsql_test 1.3 { + DELETE FROM ft WHERE rowid < 50 +} + +do_execsql_test 1.4 { + SELECT level, segment, nentry, nentrytombstone FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {0 0 1000 49} + +do_execsql_test 1.5 { + DELETE FROM ft WHERE rowid < 1000 +} + +do_execsql_test 1.6 { + SELECT level, segment, nentry, nentrytombstone FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {1 0 1 0} + +#-------------------------------------------------------------------------- +reset_db +db func document document + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); +} + +do_test 2.1 { + for {set ii 0} {$ii < 5000} {incr ii} { + execsql { INSERT INTO ft VALUES( document(12) ) } + } +} {} + +do_execsql_test 2.2 { + SELECT sum(nentry) - sum(nentrytombstone) FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {5000} + +for {set ii 5000} {$ii >= 0} {incr ii -100} { + do_execsql_test 2.3.$ii { + DELETE FROM ft WHERE rowid > $ii + } + do_execsql_test 2.3.$ii.2 { + SELECT + CAST((total(nentry) - total(nentrytombstone)) AS integer) + FROM + fts5_structure( (SELECT block FROM ft_data WHERE id=10) ) + } $ii +} + +execsql_pp { + SELECT * FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} + +do_test 2.4 { + for {set ii 0} {$ii < 5000} {incr ii} { + execsql { INSERT INTO ft VALUES( document(12) ) } + } +} {} + +for {set ii 1} {$ii <= 5000} {incr ii 10} { + do_execsql_test 2.3.$ii { + DELETE FROM ft WHERE rowid = $ii; + INSERT INTO ft VALUES( document(12) ); + INSERT INTO ft(ft, rank) VALUES('merge', -10); + } + + do_execsql_test 2.3.$ii.2 { + SELECT + CAST((total(nentry) - total(nentrytombstone)) AS integer) + FROM + fts5_structure( (SELECT block FROM ft_data WHERE id=10) ) + } 5000 +} + +#------------------------------------------------------------------------- +reset_db +db func document document +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO ft SELECT document(12) FROM s; +} + +do_catchsql_test 3.1 { + INSERT INTO ft(ft, rank) VALUES('deletemerge', 'text'); +} {1 {SQL logic error}} +do_catchsql_test 3.2 { + INSERT INTO ft(ft, rank) VALUES('deletemerge', 50); +} {0 {}} +do_execsql_test 3.3 { + SELECT * FROM ft_config WHERE k='deletemerge' +} {deletemerge 50} +do_catchsql_test 3.4 { + INSERT INTO ft(ft, rank) VALUES('deletemerge', 101); +} {0 {}} +do_execsql_test 3.5 { + SELECT * FROM ft_config WHERE k='deletemerge' +} {deletemerge 101} + +do_execsql_test 3.6 { + DELETE FROM ft WHERE rowid<95 +} + +do_execsql_test 3.7 { + SELECT nentrytombstone, nentry FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {94 100} + +do_execsql_test 3.8 { + DELETE FROM ft WHERE rowid=95 +} + +do_execsql_test 3.9 { + SELECT nentrytombstone, nentry FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {95 100} + +do_execsql_test 3.10 { + DELETE FROM ft; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO ft SELECT document(12) FROM s; + INSERT INTO ft(ft, rank) VALUES('deletemerge', 50); +} + +do_execsql_test 3.11 { + DELETE FROM ft WHERE rowid<95 +} + +do_execsql_test 3.12 { + SELECT nentrytombstone, nentry FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {0 6} + +#------------------------------------------------------------------------- +reset_db +db func document document +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x, content='', contentless_delete=1); + INSERT INTO x1(x1, rank) VALUES('usermerge', 16); + INSERT INTO x1(x1, rank) VALUES('deletemerge', 40); + INSERT INTO x1 VALUES('one'); + INSERT INTO x1 VALUES('two'); + INSERT INTO x1 VALUES('three'); + INSERT INTO x1 VALUES('four'); + INSERT INTO x1 VALUES('five'); + INSERT INTO x1 VALUES('six'); + INSERT INTO x1 VALUES('seven'); + INSERT INTO x1 VALUES('eight'); + INSERT INTO x1 VALUES('nine'); + INSERT INTO x1 VALUES('ten'); +} + +do_execsql_test 4.1 { + SELECT level, segment FROM fts5_structure(( + SELECT block FROM x1_data WHERE id=10 + )) +} { + 0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9 +} + +for {set ii 1} {$ii < 4} {incr ii} { + do_execsql_test 4.2.$ii { + DELETE FROM x1 WHERE rowid = $ii; + INSERT INTO x1(x1, rank) VALUES('merge', 5); + SELECT level, segment FROM fts5_structure(( + SELECT block FROM x1_data WHERE id=10 + )) + } { + 0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9 + } +} + +do_execsql_test 4.3 { + DELETE FROM x1 WHERE rowid = $ii; + INSERT INTO x1(x1, rank) VALUES('merge', 5); + SELECT level, segment, nentry FROM fts5_structure(( + SELECT block FROM x1_data WHERE id=10 + )) +} { + 1 0 6 +} + +finish_test + diff --git a/ext/fts5/test/fts5faultF.test b/ext/fts5/test/fts5faultF.test new file mode 100644 index 0000000000..96cc2b083f --- /dev/null +++ b/ext/fts5/test/fts5faultF.test @@ -0,0 +1,111 @@ +# 2023 July 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. Particularly those that may occur +# when using contentless_delete=1 databases. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultF + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +faultsim_save_and_close +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen +} -body { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, content=, contentless_delete=1) + } +} -test { + faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} +} + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1); + BEGIN; + INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d'); + COMMIT; + DELETE FROM t1 WHERE rowid IN (2, 4); +} + +do_faultsim_test 2 -prep { + sqlite3 db test.db + execsql { SELECT rowid FROM t1 } +} -body { + execsql { + SELECT rowid FROM t1('b c'); + } +} -test { + faultsim_test_result {0 {1 3}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1); + BEGIN; + INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d'); + COMMIT; +} + +faultsim_save_and_close +do_faultsim_test 3 -prep { + faultsim_restore_and_reopen + execsql { SELECT rowid FROM t1 } +} -body { + execsql { + INSERT INTO t1(rowid, doc) VALUES(5, 'a b c d'); + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t1(rowid, doc) SELECT i, 'a b c d' FROM s; +} + +do_execsql_test 4.1 { DELETE FROM t1 WHERE rowid <= 25 } + +faultsim_save_and_close +do_faultsim_test 4 -faults oom-t* -prep { + faultsim_restore_and_reopen + execsql { SELECT rowid FROM t1 } +} -body { + execsql { + DELETE FROM t1 WHERE rowid < 100 + } +} -test { + faultsim_test_result {0 {}} +} + + +finish_test + diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 1f366a6188..2bf619389e 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -898,6 +898,7 @@ static int sessionPreupdateEqual( rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal); } assert( rc==SQLITE_OK ); + (void)rc; /* Suppress warning about unused variable */ if( sqlite3_value_type(pVal)!=eType ) return 0; /* A SessionChange object never has a NULL value in a PK column */ diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 8fedaee7b6..3cd1d930af 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -451,6 +451,13 @@ emcc.exportedRuntimeMethods := \ emcc.jsflags += $(emcc.exportedRuntimeMethods) emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 emcc.jsflags += -sIMPORTED_MEMORY +#emcc.jsflags += -sASYNCIFY=2 +# ^^^ ASYNCIFY=2 is for experimental JSPI support +# (https://v8.dev/blog/jspi), but enabling it causes the lib-level +# init code to throw inexplicable complaints about C-level function +# signatures not matching what we expect them to be. JSPI requires, as of +# this writing, requires an experimental Chrome flag: +# chrome://flags/#enable-experimental-webassembly-stack-switching emcc.jsflags += -sSTRICT_JS=0 # STRICT_JS disabled due to: # https://github.com/emscripten-core/emscripten/issues/18610 @@ -677,10 +684,15 @@ sqlite3-wasmfs.cfiles := $(sqlite3-wasm.cfiles) # # Upstream RFE: # https://github.com/emscripten-core/emscripten/issues/18237 +# +# Maintenance reminder: Mac sed works differently than GNU sed, so +# don't use sed for this. define SQLITE3.xJS.ESM-EXPORT-DEFAULT if [ x1 = x$(1) ]; then \ echo "Fragile workaround for emscripten/issues/18237. See SQLITE3.xJS.RECIPE."; \ - sed -i -e '0,/^export default/{/^export default/d;}' $@ || exit $$?; \ + {\ + awk '/^export default/ && !f{f=1; next} 1' $@ > $@.tmp && mv $@.tmp $@; \ + } || exit $$?; \ if [ x != x$(2) ]; then \ if ! grep -q '^export default' $@; then \ echo "Cannot find export default." 1>&2; \ @@ -858,7 +870,7 @@ clean-batch: # a regular basis with different -Ox flags and rebuilding the batch # pieces each time is an unnecessary time sink. batch: batch-runner.list -all: batch +#all: batch # end batch-runner.js ######################################################################## # Wasmified speedtest1 is our primary benchmarking tool. diff --git a/ext/wasm/README.md b/ext/wasm/README.md index 769fe9edca..0c328310d9 100644 --- a/ext/wasm/README.md +++ b/ext/wasm/README.md @@ -90,7 +90,7 @@ features in the apps which use them. * Remote: Install the SQLite source tree. CD to ext/wasm * Remote: "`make`" to build WASM * Remote: `althttpd --enable-sab --port 8080 --popup` - * Local: `ssh -L 8180:remote:8080 remote` + * Local: `ssh -L 8180:localhost:8080 remote` * Local: Point your web-browser at http://localhost:8180/index.html In order to enable [SharedArrayBuffer][], the web-browser requires diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index fac00370dd..00cb3fab90 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -119,5 +119,6 @@ const toExportForESM = return globalThis.sqlite3InitModule /* required for ESM */; })(); //#if target=es6-module -export { toExportForESM as default, toExportForESM as sqlite3InitModule } +sqlite3InitModule = toExportForESM; +export default sqlite3InitModule; //#endif diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index 3661464e1b..65dbb4eb64 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -22,7 +22,9 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build */ const SABC = Object.assign( Object.create(null), { - exports: Module['asm'], + exports: ('undefined'===typeof wasmExports) + ? Module['asm']/* emscripten <=3.1.43 */ + : wasmExports /* emscripten >=3.1.44 */, memory: Module.wasmMemory /* gets set if built with -sIMPORTED_MEMORY */ }, globalThis.sqlite3ApiConfig || {} diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index f444ec975c..572efeed5c 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -728,6 +728,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Populate api object with sqlite3_...() by binding the "raw" wasm exports into type-converting proxies using wasm.xWrap(). */ + if(0 === wasm.exports.sqlite3_step.length){ + /* This environment wraps exports in nullary functions, which means + we must disable the arg-count validation we otherwise perform + on the wrappers. */ + wasm.xWrap.doArgcCheck = false; + sqlite3.config.warn( + "Disabling sqlite3.wasm.xWrap.doArgcCheck due to environmental quirks." + ); + } for(const e of wasm.bindingSignatures){ capi[e[0]] = wasm.xWrap.apply(null, e); } diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 9bf1421a9f..ee8d1a5d68 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -501,6 +501,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* Current number of in-use files from pool. */ getFileCount(){return this.#mapFilenameToSAH.size} + /* Returns an array of the names of all + currently-opened client-specified filenames. */ + getFileNames(){ + const rc = []; + const iter = this.#mapFilenameToSAH.keys(); + for(const n of iter) rc.push(n); + return rc; + } + // #createFileObject(sah,clientName,opaqueName){ // const f = Object.assign(Object.create(null),{ // clientName, opaqueName @@ -901,6 +910,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ getCapacity(){ return this.#p.getCapacity(this.#p) } getFileCount(){ return this.#p.getFileCount() } + getFileNames(){ return this.#p.getFileNames() } async reserveMinimumCapacity(min){ const c = this.#p.getCapacity(); @@ -1059,6 +1069,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Returns the number of files from the pool currently allocated to slots. This is not the same as the files being "opened". + - array getFileNames() + + Returns an array of the names of the files currently allocated to + slots. This list is the same length as getFileCount(). + - void importDb(name, byteArray) Imports the contents of an SQLite database, provided as a byte diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index f48e8a7d21..ee7ea20c34 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -111,7 +111,8 @@ of `target.instance` (a WebAssembly.Module instance) and it must contain the symbols exported by the WASM module associated with this code. In an Enscripten environment it must be set to - `Module['asm']`. The exports object must contain a minimum of the + `Module['asm']` (versions <=3.1.43) or `wasmExports` (versions + >=3.1.44). The exports object must contain a minimum of the following symbols: - `memory`: a WebAssembly.Memory object representing the WASM diff --git a/ext/wasm/demo-123.js b/ext/wasm/demo-123.js index 311afcc827..8e03ee80fa 100644 --- a/ext/wasm/demo-123.js +++ b/ext/wasm/demo-123.js @@ -235,10 +235,10 @@ - get change count (total or statement-local, 32- or 64-bit) - get a DB's file name - + Misc. Stmt features: - - Various forms of bind() + - Various forms of bind() - clearBindings() - reset() - Various forms of step() diff --git a/ext/wasm/index.html b/ext/wasm/index.html index d91dca6cb4..70ce0441e0 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -107,7 +107,8 @@
  • module-symbols gives a high-level overview of the symbols exposed by the JS module.
    • -
  • batch-runner: runs batches of SQL exported from speedtest1.
    • +
  • test-opfs-vfs (same with verbose output and sanity-checking tests) is an diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 905340b234..db4b50f4b6 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3072,10 +3072,23 @@ globalThis.sqlite3InitModule = sqlite3InitModule; db = new u2.OpfsSAHPoolDb(dbName); T.assert(1 === u1.getFileCount()); db.close(); - T.assert(1 === u1.getFileCount()) + const fileNames = u1.getFileNames(); + T.assert(1 === fileNames.length) + .assert(dbName === fileNames[0]) + .assert(1 === u1.getFileCount()) .assert(true === u1.unlink(dbName)) .assert(false === u1.unlink(dbName)) - .assert(0 === u1.getFileCount()); + .assert(0 === u1.getFileCount()) + .assert(0 === u1.getFileNames().length); + + // Demonstrate that two SAH pools can coexist so long as + // they have different names. + const conf2 = JSON.parse(JSON.stringify(sahPoolConfig)); + conf2.name += '-test2'; + const POther = await inst(conf2); + log("Installed second SAH instance as",conf2.name); + T.assert(0 === POther.getFileCount()) + .assert(true === await POther.removeVfs()); if(0){ /* Enable this block to inspect vfs's contents via the dev console or OPFS Explorer browser extension. The @@ -3087,7 +3100,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)); let cErr, u3; - const conf2 = JSON.parse(JSON.stringify(sahPoolConfig)); conf2.$testThrowInInit = new Error("Testing throwing during init."); conf2.name = sahPoolConfig.name+'-err'; const P3 = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e); diff --git a/manifest b/manifest index 8454917dcc..ca75d43b3f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ C Merge\sthe\slatest\strunk\senhancements\sinto\sthe\sbegin-concurrent\sbranch. -D 2023-07-24T13:15:05.171 +D 2023-07-28T18:43:28.966 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -87,15 +87,15 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a -F ext/fts5/fts5Int.h ed48a096418ff4a7c02ac9bd1e8d40c46de21b79a132b8b08d3f32233703de7d +F ext/fts5/fts5Int.h 78a63cc0795186cde5384816a9403a68c65774b35d952e05b81a1b4b158e07c8 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 -F ext/fts5/fts5_config.c 051056a9052f5d3a4d1c695f996fd364f920e341f136c60ab2c04aa7e267113f -F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1 -F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 430d3edfc5b51d02f0671e2b8268c12bf485f7dc80db2b61c996e74814bc46b0 -F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d -F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 +F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081 +F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d +F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d +F ext/fts5/fts5_index.c a4e35cd126c19df66887801db15ccae7290b51129add3d4cdd31e2d9ea240398 +F ext/fts5/fts5_main.c 2f87ee44fdb21539c264541149f07f70e065d58f37420063e5ddef80ba0f5ede +F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff @@ -133,6 +133,10 @@ F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825 F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4 F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283 +F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9 +F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7 +F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286 +F ext/fts5/test/fts5contentless4.test 0f43ededc2874f65d7da99b641a82239854d98d3fa43db729f284b723f23b69f F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78 @@ -160,6 +164,7 @@ F ext/fts5/test/fts5faultA.test be4487576bff8c22cee6597d1893b312f306504a8c6ccd3c F ext/fts5/test/fts5faultB.test d606bdb8e81aaeb6f41de3fc9fc7ae315733f0903fbff05cf54f5b045b729ab5 F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860bbf9e504b9647996 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075 +F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -480,34 +485,34 @@ F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb4 F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc F ext/session/sqlite3changebatch.c d5553b79e012ee2cb06c0a96bdf9dfe19e66354390ea0036cc46c4953142d517 F ext/session/sqlite3changebatch.h e72016998c9a22d439ddfd547b69e1ebac810c24 -F ext/session/sqlite3session.c 3f88516da48852eff1643f4c367e9b65f75c1904d041e8f4a84c0fcda3d8379a +F ext/session/sqlite3session.c a9ec768427aeccc2441b03775be3afc591e8e01db01f77af6af98b578bd5717a F ext/session/sqlite3session.h 0acea22732ef17c1d56d68cdd70948101e7ee2053e92764a80051d9572d432d1 F ext/session/test_session.c 8957ef47655196b0d94cde7095b294fe34d7bac25c5dc7c4bb21a0b6e4f991c7 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 -F ext/wasm/GNUmakefile 4e8260d05c52d9924b853efbdfe052bd483cfe42f055567c1bbf29d274794b22 +F ext/wasm/GNUmakefile ddf1aede4275e404c7eda782462c33b6406fcd2dd327241f6b22c0f7b80938e4 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 -F ext/wasm/README.md 0895244c0539ae68cf8c70d59c2de512532fd47cfba313268e2b672e6359112e +F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 5eb44fa02e9c693a1884a3692428647894b0380b24bca120866b7a24c8786134 -F ext/wasm/api/extern-post-js.c-pp.js 116749b7e55b7519129de06d3d353e19df68cfb24b12204aa4dc30c9a83023fe +F ext/wasm/api/extern-post-js.c-pp.js e7257ea56d4077d0773d7537fcb8eade59aad0c118dab92497c27939edca41cb F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 -F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf -F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 +F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e +F ext/wasm/api/sqlite3-api-glue.js cc6b0bb093bdb6279d4af259200b7b9e150e3796a8a3a4cd09a4928c43d25e56 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 F ext/wasm/api/sqlite3-api-prologue.js cbd7d6ba185f3a844a8b0020e954b49bbc2ca78b305d117bec2ceca21431795a F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 7931b50b63246a3d6b46a4c703c28820aa10c5b1ae7c0718e1f58dae2cf6db85 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 44740935f6eeab3a0c6dc9f0ca93aebb6339f8d8365f03ea6d7808c134dbfe46 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e7a690e0e78ff4d563f2eca468f91db69f001ff4b79c6d2304cbb6f62dca437d F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js c5ac33e39f21a3481812d7333ca6e18853640d423a01960ca8dbc6e7c5c3c21c @@ -518,10 +523,10 @@ F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f -F ext/wasm/common/whwasmutil.js ae263dec9d7384f4c530f324b99d00516a4d6f26424372daee65031e00eb49b3 +F ext/wasm/common/whwasmutil.js db6368ee57af90ee6691b6fb3ca97ee8064d12482e06a29113127c67b08f956e F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 -F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 +F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5 F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8 F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f @@ -536,12 +541,12 @@ F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa0 F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e -F ext/wasm/index.html b4e55de741be9fb7656445ea55085f703a784aebde620e1c4852fa21c1ac1c5b +F ext/wasm/index.html 4e7847b909f4ae0da8c829b150b79454050e53b3658431f138636257729cd42b F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10 -F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 w ext/wasm/scratchpad-wasmfs-main.html -F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 w ext/wasm/scratchpad-wasmfs-main.js +F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 +F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5 @@ -554,7 +559,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js f835c9f703b562142f23a3607fa4a34cb6aece5fb5d674ea5bd7d37b0e47e104 +F ext/wasm/tester1.c-pp.js b9a493a764f2113f6a8205d4e5573a25731d6f965338005460920389ac0693e0 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -582,8 +587,8 @@ F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 501daeef838fa82a9fb53540d72f29e3d9172c8867f1e19f94f681e2e20b966e F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 -F src/btree.c c6f426981843cf31116d45ea619c875657c2fdbb089424b3374c1ff2499fc4ec -F src/btree.h 77a092acf63526827e74e88d0480123212d079593a841ff1fe85507adf256ef6 +F src/btree.c 3504c745ece8ffb5335a14f72b1c34484e66c14d5c3b56b3e1cbed7fa02dfc4c +F src/btree.h d906e4d53f483c83d471d99479fa73fcdf20696305d578876f46ee283f3507cb F src/btreeInt.h c490cf859feeec542d60e6df7976dc8f99a82c587a8a96e432be66b3a08fc83c F src/build.c 84238806b9a9ca5373ccaea65a86129717d023bc582919f7f5a91d1b92e1a4c1 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 @@ -603,7 +608,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 46ea5566e1363f4f353db79b0378c2bf8ffdf9d4667daee3df67b14669767bed +F src/json.c 60b6b0815ddaf546396e3c7328065a114b581721dc1fa20697a52610fb2259aa F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 429a1a25752edd6afa97de4b8ac10b98f557bbd93f01e39fcd2fa2204a8f54bb @@ -632,28 +637,28 @@ F src/os_win.c 7038223a1cda0a47e2ab4db47f63bf1833fe53ba0542f0f283a062ea13894103 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c e2039b61af0fdb3e5c4a476b064d20a2c7a41eb86e41de1c03a8093fb28cbcaa F src/pager.h 7b2ec7bba30b21a97b68d5bdc0dbb82a75f48c4b1457180988f9d409fb789e16 -F src/parse.y cf35577daeb17122be7c42d41485249360d19d3c8a40496872ea8c0a200cfa44 +F src/parse.y 92a9cc670816e1274a107d02ed8efec6028c23713c767f035479fde411c86f27 F src/pcache.c 4cd4a0043167da9ba7e19b4d179a0e6354e7fe32c16f781ecf9bf0a5ff63b40b F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 F src/pragma.c 37b8fb02d090262280c86e1e2654bf59d8dbfbfe8dc6733f2b968a11374c095a F src/pragma.h 1f421360eed1a7721e8c521463df8519a7c8d0d5893ebd9dbfe0dba8de996f8c -F src/prepare.c d6c4354f8ea0dc06962fbabc4b68c4471a45276a2918c929be00f9f537f69eb1 -F src/printf.c 84b7b4b647f336934a5ab2e7f0c52555833cc0778d2d60e016cca52ee8c6cd8f +F src/prepare.c 80548297dc0e1fb3139cdebffb5a1bcac3dfac66d791012dd74838e70445072d +F src/printf.c e3ba080e2f409f9bfcc8d34724e6fc160e9c718dc92d0548f6b71b8b6f860ce2 F src/random.c a3e70f8515721ff24d2c0e6afd83923e8faab5ab79ececea4c1bf9fe4049fbb2 F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 8425355cfbe559fb00a1c7f251daf629cb433c751efc550af5305551300d1462 -F src/shell.c.in d320d8a13636de06d777cc1eab981caca304e175464e98183cf4ea68d93db818 -F src/sqlite.h.in 0b25130ef3c980630ad32e022c5671c371dc75765358a98d915e1b8aa7896e2e +F src/select.c 0b7c7b89576c39a5e6c166a12d1f377fd58bd26da16db0a580bbfa74d7b9e140 +F src/shell.c.in 694aaf751f00610381533d4a31c83d142cfc83ef91ef65e2aa6912ace7c39b40 +F src/sqlite.h.in 32d34a790dfd7b5e4daa5a83b93da0a3434234108e17086f1c63f20fda38dc31 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 52017f11ca08bf20da0f0cc6fb5188ffb99389ffa67b3f071fe952148b84a5be +F src/sqliteInt.h cb89365fd2c829c4f549a097640943f8be9bb9136986d176144d7ffa4b2de3b8 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c ecbc3c99c0d0c3ed122a913f143026c26d38d57f33e06bb71185dd5c1efe37cd -F src/test1.c df075b328166df5a0e59428033d85a4a83c614062641b10fd51e7cbbcf9810f7 +F src/test1.c fd08bf60687514a33f21561cfe43036a97938baa4d1752064c8b78d468162781 F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef F src/test3.c e5178558c41ff53236ae0271e9acb3d6885a94981d2eb939536ee6474598840e F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 @@ -713,16 +718,16 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c c2aa170f2eb429235b1dddce8952770787ffa5124dc89d405bfbe8ebad8e7ebd F src/vacuum.c b1dd6d73869229b6e08bac910ac011dc9da42e3120ec2b7241accc5a752bd419 -F src/vdbe.c c0c9aaf87809ff860ee8b33c0cb704ad03e89f80f515ac836e61717d2b5184d3 +F src/vdbe.c 906cfa414798df2ce5d917919492435ad9f92a577503d3a5a94bcf5ef37d18b9 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 -F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 -F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 -F src/vdbeaux.c 7b6d7aa0667a381c4aa1414b531dad7a8480e08cad1cb19c6fd38908c1790036 +F src/vdbeInt.h cd829ff67fc18aff2476fe3209262ba8671cb43d51cb6913de4a670e37d2fbe3 +F src/vdbeapi.c 0cba9fe6e860e46bfb5e45a76a8c2c6ccd5ebc8f954319e4288e7793c2eca88c +F src/vdbeaux.c e4ff54897dca508b3c3726973a1cb08f09d81cbe13c0c36012d9c2572402327c F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c 40afb83ed848e235848ffdd3ba25adca4ba602111b8ed3b05ae3b1b12e0eacee +F src/vdbemem.c adb5877dc54330865e45e0fba50f6455c1d9d8b83434c59f0e3d2d2a0b9f0e58 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 -F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac +F src/vdbevtab.c 57fa8f56478e5b5cb558cb425e7878515e0a105c54f96f1d1bbf4b9433529254 F src/vtab.c 1ecf8c3745d29275688d583e12822fa984d421e0286b5ef50c137bc3bf6d7a64 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 5a1e2b889af681ff4707c4100dcda7b24ac0939fdb3e793c1109bad995ed6f94 @@ -1147,7 +1152,7 @@ F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5b F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba F test/fuzzdata4.db b502c7d5498261715812dd8b3c2005bad08b3a26e6489414bd13926cd3e42ed2 F test/fuzzdata5.db e35f64af17ec48926481cfaf3b3855e436bd40d1cfe2d59a9474cb4b748a52a5 -F test/fuzzdata6.db 92a80e4afc172c24f662a10a612d188fb272de4a9bd19e017927c95f737de6d7 +F test/fuzzdata6.db b8725a5f5cf7a3b7241a9038e57ca7e7cc8c3f4d86b44bd770617bda245ab2b0 F test/fuzzdata7.db 0166b56fd7a6b9636a1d60ef0a060f86ddaecf99400a666bb6e5bbd7199ad1f2 F test/fuzzdata8.db f6c2f2af4deaaae0ddb3310d509c2659990794aa653dc501b80a0534c3493f80 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 @@ -1239,9 +1244,9 @@ F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ff F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json/README.md 0992b8ccbecd8424b84c1173f9ac56bcc0ae96d49912fd2d86c04eb1f3cba7af +F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f -F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d +F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x F test/json101.test 94126d4291d4a00e45f6988ce885c410de69243490e46e70e9946cb6e6f9ea02 F test/json102.test 13dc9e7b7f359ecb861e02f9bd7019f7342a63d1c354273b0a8f3904050560a8 @@ -2061,8 +2066,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3d0823c48eb8d91b747e896ee416da22f60456f14de51ff5cf31657d94c343f3 c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3 -R 9668f130da5a6db61f0451dff404aecb +P 967be041ea97ad1e96f46e18330e6477fdda125c83345f15e7c5502a41432c57 06eae046c1a1155f22590b88942db72f01482f2363da9753f8adfdb322679055 0443c0ef85f8f6f3efdee68bf59126c9459f08a58458756f8cbeeaf043d43a86 +R 03db566bae4052a793882cf96023c74e U drh -Z 83d0046c9cc5d1549c18f726845aa17c +Z 270e1c3de14bf94286c9440395b93cd4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f34dc7a68b..a69f7afef0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -967be041ea97ad1e96f46e18330e6477fdda125c83345f15e7c5502a41432c57 \ No newline at end of file +7407dda8b4c704cff665ba06108ea33c4099ff6446357020e5e8ecad0c86ed57 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 554cda10f7..5d9cbd7f89 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5277,7 +5277,6 @@ void sqlite3BtreeCursorUnpin(BtCursor *pCur){ pCur->curFlags &= ~BTCF_Pinned; } -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC /* ** Return the offset into the database file for the start of the ** payload to which the cursor is pointing. @@ -5289,7 +5288,6 @@ i64 sqlite3BtreeOffset(BtCursor *pCur){ return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) + (i64)(pCur->info.pPayload - pCur->pPage->aData); } -#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ /* ** Return the number of bytes of payload for the entry that pCur is diff --git a/src/btree.h b/src/btree.h index c1e24ffed4..d01ebc7c09 100644 --- a/src/btree.h +++ b/src/btree.h @@ -321,9 +321,7 @@ int sqlite3BtreePrevious(BtCursor*, int flags); i64 sqlite3BtreeIntegerKey(BtCursor*); void sqlite3BtreeCursorPin(BtCursor*); void sqlite3BtreeCursorUnpin(BtCursor*); -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC i64 sqlite3BtreeOffset(BtCursor*); -#endif int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); u32 sqlite3BtreePayloadSize(BtCursor*); diff --git a/src/json.c b/src/json.c index 05046b5b54..cc39264c05 100644 --- a/src/json.c +++ b/src/json.c @@ -59,6 +59,7 @@ static const char jsonIsSpace[] = { typedef struct JsonString JsonString; typedef struct JsonNode JsonNode; typedef struct JsonParse JsonParse; +typedef struct JsonCleanup JsonCleanup; /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator @@ -74,16 +75,26 @@ struct JsonString { char zSpace[100]; /* Initial static space */ }; +/* A deferred cleanup task. A list of JsonCleanup objects might be +** run when the JsonParse object is destroyed. +*/ +struct JsonCleanup { + JsonCleanup *pJCNext; /* Next in a list */ + void (*xOp)(void*); /* Routine to run */ + void *pArg; /* Argument to xOp() */ +}; + /* JSON type values */ -#define JSON_NULL 0 -#define JSON_TRUE 1 -#define JSON_FALSE 2 -#define JSON_INT 3 -#define JSON_REAL 4 -#define JSON_STRING 5 -#define JSON_ARRAY 6 -#define JSON_OBJECT 7 +#define JSON_SUBST 0 /* Special edit node. Uses u.iPrev */ +#define JSON_NULL 1 +#define JSON_TRUE 2 +#define JSON_FALSE 3 +#define JSON_INT 4 +#define JSON_REAL 5 +#define JSON_STRING 6 +#define JSON_ARRAY 7 +#define JSON_OBJECT 8 /* The "subtype" set for JSON values */ #define JSON_SUBTYPE 74 /* Ascii for "J" */ @@ -92,52 +103,87 @@ struct JsonString { ** Names of the various JSON types: */ static const char * const jsonType[] = { + "subst", "null", "true", "false", "integer", "real", "text", "array", "object" }; /* Bit values for the JsonNode.jnFlag field */ -#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ -#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ -#define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ -#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ -#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x40 /* Is a label of an object */ -#define JNODE_JSON5 0x80 /* Node contains JSON5 enhancements */ +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ +#define JNODE_REMOVE 0x04 /* Do not output */ +#define JNODE_REPLACE 0x08 /* Target of a JSON_SUBST node */ +#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x20 /* Is a label of an object */ +#define JNODE_JSON5 0x40 /* Node contains JSON5 enhancements */ -/* A single node of parsed JSON +/* A single node of parsed JSON. An array of these nodes describes +** a parse of JSON + edits. +** +** Use the json_parse() SQL function (available when compiled with +** -DSQLITE_DEBUG) to see a dump of complete JsonParse objects, including +** a complete listing and decoding of the array of JsonNodes. */ struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 jnFlags; /* JNODE flags */ u8 eU; /* Which union element to use */ - u32 n; /* Bytes of content, or number of sub-nodes */ + u32 n; /* Bytes of content for INT, REAL or STRING + ** Number of sub-nodes for ARRAY and OBJECT + ** Node that SUBST applies to */ union { const char *zJContent; /* 1: Content for INT, REAL, and STRING */ u32 iAppend; /* 2: More terms for ARRAY and OBJECT */ u32 iKey; /* 3: Key for ARRAY objects in json_tree() */ - u32 iReplace; /* 4: Replacement content for JNODE_REPLACE */ - JsonNode *pPatch; /* 5: Node chain of patch for JNODE_PATCH */ + u32 iPrev; /* 4: Previous SUBST node, or 0 */ } u; }; -/* A completely parsed JSON string + +/* A parsed and possibly edited JSON string. Lifecycle: +** +** 1. JSON comes in and is parsed into an array aNode[]. The original +** JSON text is stored in zJson. +** +** 2. Zero or more changes are made (via json_remove() or json_replace() +** or similar) to the aNode[] array. +** +** 3. A new, edited and mimified JSON string is generated from aNode +** and stored in zAlt. The JsonParse object always owns zAlt. +** +** Step 1 always happens. Step 2 and 3 may or may not happen, depending +** on the operation. +** +** aNode[].u.zJContent entries typically point into zJson. Hence zJson +** must remain valid for the lifespan of the parse. For edits, +** aNode[].u.zJContent might point to malloced space other than zJson. +** Entries in pClup are responsible for freeing that extra malloced space. +** +** When walking the parse tree in aNode[], edits are ignored if useMod is +** false. */ struct JsonParse { u32 nNode; /* Number of slots of aNode[] used */ u32 nAlloc; /* Number of slots of aNode[] allocated */ JsonNode *aNode; /* Array of nodes containing the parse */ - const char *zJson; /* Original JSON string */ + char *zJson; /* Original JSON string (before edits) */ + char *zAlt; /* Revised and/or mimified JSON */ u32 *aUp; /* Index of parent of each node */ + JsonCleanup *pClup;/* Cleanup operations prior to freeing this object */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ + u8 bJsonIsRCStr; /* True if zJson is an RCStr */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ + u8 useMod; /* Actually use the edits contain inside aNode */ + u8 hasMod; /* aNode contains edits from the original zJson */ + u32 nJPRef; /* Number of references to this object */ int nJson; /* Length of the zJson string in bytes */ + int nAlt; /* Length of alternative JSON string zAlt, in bytes */ u32 iErr; /* Error location in zJson[] */ - u32 iHold; /* Replace cache line with the lowest iHold value */ + u32 iSubst; /* Last JSON_SUBST entry in aNode[] */ + u32 iHold; /* Age of this entry in the cache for LRU replacement */ }; /* @@ -170,17 +216,15 @@ static void jsonInit(JsonString *p, sqlite3_context *pCtx){ jsonZero(p); } - /* Free all allocated memory and reset the JsonString object back to its ** initial state. */ static void jsonReset(JsonString *p){ - if( !p->bStatic ) sqlite3_free(p->zBuf); + if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf); jsonZero(p); } - -/* Report an out-of-memory (OOM) condition +/* Report an out-of-memory (OOM) condition */ static void jsonOom(JsonString *p){ p->bErr = 1; @@ -196,7 +240,7 @@ static int jsonGrow(JsonString *p, u32 N){ char *zNew; if( p->bStatic ){ if( p->bErr ) return 1; - zNew = sqlite3_malloc64(nTotal); + zNew = sqlite3RCStrNew(nTotal); if( zNew==0 ){ jsonOom(p); return SQLITE_NOMEM; @@ -205,12 +249,12 @@ static int jsonGrow(JsonString *p, u32 N){ p->zBuf = zNew; p->bStatic = 0; }else{ - zNew = sqlite3_realloc64(p->zBuf, nTotal); - if( zNew==0 ){ - jsonOom(p); + p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); + if( p->zBuf==0 ){ + p->bErr = 1; + jsonZero(p); return SQLITE_NOMEM; } - p->zBuf = zNew; } p->nAlloc = nTotal; return SQLITE_OK; @@ -273,6 +317,24 @@ static void jsonAppendChar(JsonString *p, char c){ } } +/* Try to force the string to be a zero-terminated RCStr string. +** +** Return true on success. Return false if an OOM prevents this +** from happening. +*/ +static int jsonForceRCStr(JsonString *p){ + jsonAppendChar(p, 0); + if( p->bErr ) return 0; + p->nUsed--; + if( p->bStatic==0 ) return 1; + p->nAlloc = 0; + p->nUsed++; + jsonGrow(p, p->nUsed); + p->nUsed--; + return p->bStatic==0; +} + + /* Append a comma separator to the output buffer, if the previous ** character is not '[' or '{'. */ @@ -344,7 +406,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendRawNZ(p, zIn, i); zIn += i; N -= i; - if( N==0 ) break; + if( N==0 ) break; } assert( zIn[0]=='\\' ); switch( (u8)zIn[1] ){ @@ -453,7 +515,7 @@ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ /* -** Append a function parameter value to the JSON string under +** Append a function parameter value to the JSON string under ** construction. */ static void jsonAppendValue( @@ -498,16 +560,25 @@ static void jsonAppendValue( /* Make the JSON in p the result of the SQL function. +** +** The JSON string is reset. */ static void jsonResult(JsonString *p){ if( p->bErr==0 ){ - jsonAppendChar(p, 0); - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed-1, - p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, - SQLITE_UTF8); - jsonZero(p); + if( p->bStatic ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + }else if( jsonForceRCStr(p) ){ + sqlite3RCStrRef(p->zBuf); + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + (void(*)(void*))sqlite3RCStrUnref, + SQLITE_UTF8); + } } - assert( p->bStatic ); + if( p->bErr==1 ){ + sqlite3_result_error_nomem(p->pCtx); + } + jsonReset(p); } /************************************************************************** @@ -532,20 +603,73 @@ static u32 jsonNodeSize(JsonNode *pNode){ ** delete the JsonParse object itself. */ static void jsonParseReset(JsonParse *pParse){ - sqlite3_free(pParse->aNode); - pParse->aNode = 0; + while( pParse->pClup ){ + JsonCleanup *pTask = pParse->pClup; + pParse->pClup = pTask->pJCNext; + pTask->xOp(pTask->pArg); + sqlite3_free(pTask); + } + assert( pParse->nJPRef<=1 ); + if( pParse->aNode ){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + } pParse->nNode = 0; pParse->nAlloc = 0; - sqlite3_free(pParse->aUp); - pParse->aUp = 0; + if( pParse->aUp ){ + sqlite3_free(pParse->aUp); + pParse->aUp = 0; + } + if( pParse->bJsonIsRCStr ){ + sqlite3RCStrUnref(pParse->zJson); + pParse->zJson = 0; + pParse->bJsonIsRCStr = 0; + } + if( pParse->zAlt ){ + sqlite3RCStrUnref(pParse->zAlt); + pParse->zAlt = 0; + } } /* ** Free a JsonParse object that was obtained from sqlite3_malloc(). +** +** Note that destroying JsonParse might call sqlite3RCStrUnref() to +** destroy the zJson value. The RCStr object might recursively invoke +** JsonParse to destroy this pParse object again. Take care to ensure +** that this recursive destructor sequence terminates harmlessly. */ static void jsonParseFree(JsonParse *pParse){ - jsonParseReset(pParse); - sqlite3_free(pParse); + if( pParse->nJPRef>1 ){ + pParse->nJPRef--; + }else{ + jsonParseReset(pParse); + sqlite3_free(pParse); + } +} + +/* +** Add a cleanup task to the JsonParse object. +** +** If an OOM occurs, the cleanup operation happens immediately +** and this function returns SQLITE_NOMEM. +*/ +static int jsonParseAddCleanup( + JsonParse *pParse, /* Add the cleanup task to this parser */ + void(*xOp)(void*), /* The cleanup task */ + void *pArg /* Argument to the cleanup */ +){ + JsonCleanup *pTask = sqlite3_malloc64( sizeof(*pTask) ); + if( pTask==0 ){ + pParse->oom = 1; + xOp(pArg); + return SQLITE_ERROR; + } + pTask->pJCNext = pParse->pClup; + pParse->pClup = pTask; + pTask->xOp = xOp; + pTask->pArg = pArg; + return SQLITE_OK; } /* @@ -554,19 +678,25 @@ static void jsonParseFree(JsonParse *pParse){ ** the number of JsonNode objects that are encoded. */ static void jsonRenderNode( + JsonParse *pParse, /* the complete parse of the JSON */ JsonNode *pNode, /* The node to render */ - JsonString *pOut, /* Write JSON here */ - sqlite3_value **aReplace /* Replacement values */ + JsonString *pOut /* Write JSON here */ ){ assert( pNode!=0 ); - if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ - if( (pNode->jnFlags & JNODE_REPLACE)!=0 && ALWAYS(aReplace!=0) ){ - assert( pNode->eU==4 ); - jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); - return; + while( (pNode->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ + u32 idx = (u32)(pNode - pParse->aNode); + u32 i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( inNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==4 ); + assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ + pNode = &pParse->aNode[i+1]; + break; + } + i = pParse->aNode[i].u.iPrev; } - assert( pNode->eU==5 ); - pNode = pNode->u.pPatch; } switch( pNode->eType ){ default: { @@ -625,15 +755,16 @@ static void jsonRenderNode( jsonAppendChar(pOut, '['); for(;;){ while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); + jsonRenderNode(pParse, &pNode[j], pOut); } j += jsonNodeSize(&pNode[j]); } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pNode->eU==2 ); - pNode = &pNode[pNode->u.iAppend]; + pNode = &pParse->aNode[pNode->u.iAppend]; j = 1; } jsonAppendChar(pOut, ']'); @@ -644,17 +775,18 @@ static void jsonRenderNode( jsonAppendChar(pOut, '{'); for(;;){ while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); + jsonRenderNode(pParse, &pNode[j], pOut); jsonAppendChar(pOut, ':'); - jsonRenderNode(&pNode[j+1], pOut, aReplace); + jsonRenderNode(pParse, &pNode[j+1], pOut); } j += 1 + jsonNodeSize(&pNode[j+1]); } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pNode->eU==2 ); - pNode = &pNode[pNode->u.iAppend]; + pNode = &pParse->aNode[pNode->u.iAppend]; j = 1; } jsonAppendChar(pOut, '}'); @@ -667,15 +799,26 @@ static void jsonRenderNode( ** Return a JsonNode and all its descendants as a JSON string. */ static void jsonReturnJson( + JsonParse *pParse, /* The complete JSON */ JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ + int bGenerateAlt /* Also store the rendered text in zAlt */ ){ JsonString s; - jsonInit(&s, pCtx); - jsonRenderNode(pNode, &s, aReplace); - jsonResult(&s); - sqlite3_result_subtype(pCtx, JSON_SUBTYPE); + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( pParse->nErr==0 ){ + jsonInit(&s, pCtx); + jsonRenderNode(pParse, pNode, &s); + if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ + pParse->zAlt = sqlite3RCStrRef(s.zBuf); + pParse->nAlt = s.nUsed; + } + jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); + } } /* @@ -713,9 +856,9 @@ static u32 jsonHexToInt4(const char *z){ ** Make the JsonNode the return value of the function. */ static void jsonReturn( + JsonParse *pParse, /* Complete JSON parse tree */ JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ + sqlite3_context *pCtx /* Return value for this function */ ){ switch( pNode->eType ){ default: { @@ -736,8 +879,7 @@ static void jsonReturn( int rc; int bNeg = 0; const char *z; - - + assert( pNode->eU==1 ); z = pNode->u.zJContent; if( z[0]=='-' ){ z++; bNeg = 1; } @@ -862,7 +1004,7 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - jsonReturnJson(pNode, pCtx, aReplace); + jsonReturnJson(pParse, pNode, pCtx, 0); break; } } @@ -884,6 +1026,12 @@ static int jsonParseAddNode(JsonParse*,u32,u32,const char*); #endif +/* +** Add a single node to pParse->aNode after first expanding the +** size of the aNode array. Return the index of the new node. +** +** If an OOM error occurs, set pParse->oom and return -1. +*/ static JSON_NOINLINE int jsonParseAddNodeExpand( JsonParse *pParse, /* Append the node to this object */ u32 eType, /* Node type */ @@ -900,7 +1048,7 @@ static JSON_NOINLINE int jsonParseAddNodeExpand( pParse->oom = 1; return -1; } - pParse->nAlloc = nNew; + pParse->nAlloc = sqlite3_msize(pNew)/sizeof(JsonNode); pParse->aNode = pNew; assert( pParse->nNodenAlloc ); return jsonParseAddNode(pParse, eType, n, zContent); @@ -931,6 +1079,50 @@ static int jsonParseAddNode( return pParse->nNode++; } +/* +** Add an array of new nodes to the current pParse->aNode array. +** Return the index of the first node added. +** +** If an OOM error occurs, set pParse->oom. +*/ +static void jsonParseAddNodeArray( + JsonParse *pParse, /* Append the node to this object */ + JsonNode *aNode, /* Array of nodes to add */ + u32 nNode /* Number of elements in aNew */ +){ + if( pParse->nNode + nNode > pParse->nAlloc ){ + u32 nNew = pParse->nNode + nNode; + JsonNode *aNew = sqlite3_realloc64(pParse->aNode, nNew*sizeof(JsonNode)); + if( aNew==0 ){ + pParse->oom = 1; + return; + } + pParse->nAlloc = sqlite3_msize(aNew)/sizeof(JsonNode); + pParse->aNode = aNew; + } + memcpy(&pParse->aNode[pParse->nNode], aNode, nNode*sizeof(JsonNode)); + pParse->nNode += nNode; +} + +/* +** Add a new JSON_SUBST node. The node immediately following +** this new node will be the substitute content for iNode. +*/ +static int jsonParseAddSubstNode( + JsonParse *pParse, /* Add the JSON_SUBST here */ + u32 iNode /* References this node */ +){ + int idx = jsonParseAddNode(pParse, JSON_SUBST, iNode, 0); + if( pParse->oom ) return -1; + pParse->aNode[iNode].jnFlags |= JNODE_REPLACE; + pParse->aNode[idx].eU = 4; + pParse->aNode[idx].u.iPrev = pParse->iSubst; + pParse->iSubst = idx; + pParse->hasMod = 1; + pParse->useMod = 1; + return idx; +} + /* ** Return true if z[] begins with 2 (or more) hexadecimal digits */ @@ -1097,7 +1289,7 @@ static const struct NanInfName { ** ** Special return values: ** -** 0 End if input +** 0 End of input ** -1 Syntax error ** -2 '}' seen ** -3 ']' seen @@ -1274,23 +1466,23 @@ json_parse_restart: cDelim = z[i]; for(j=i+1; 1; j++){ static const char aOk[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }; if( aOk[(unsigned char)z[j]] ) continue; c = z[j]; @@ -1556,20 +1748,18 @@ json_parse_restart: /* ** Parse a complete JSON string. Return 0 on success or non-zero if there -** are any errors. If an error occurs, free all memory associated with -** pParse. +** are any errors. If an error occurs, free all memory held by pParse, +** but not pParse itself. ** -** pParse is uninitialized when this routine is called. +** pParse must be initialized to an empty parse object prior to calling +** this routine. */ static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ - sqlite3_context *pCtx, /* Report errors here */ - const char *zJson /* Input JSON text to be parsed */ + sqlite3_context *pCtx /* Report errors here */ ){ int i; - memset(pParse, 0, sizeof(*pParse)); - if( zJson==0 ) return 1; - pParse->zJson = zJson; + const char *zJson = pParse->zJson; i = jsonParseValue(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ @@ -1598,6 +1788,7 @@ static int jsonParse( return 0; } + /* Mark node i of pParse as being a child of iParent. Call recursively ** to fill in all the descendants of node i. */ @@ -1647,35 +1838,49 @@ static int jsonParseFindParents(JsonParse *pParse){ #define JSON_CACHE_SZ 4 /* Max number of cache entries */ /* -** Obtain a complete parse of the JSON found in the first argument -** of the argv array. Use the sqlite3_get_auxdata() cache for this -** parse if it is available. If the cache is not available or if it -** is no longer valid, parse the JSON again and return the new parse, -** and also register the new parse so that it will be available for +** Obtain a complete parse of the JSON found in the pJson argument +** +** Use the sqlite3_get_auxdata() cache to find a preexisting parse +** if it is available. If the cache is not available or if it +** is no longer valid, parse the JSON again and return the new parse. +** Also register the new parse so that it will be available for ** future sqlite3_get_auxdata() calls. ** ** If an error occurs and pErrCtx!=0 then report the error on pErrCtx ** and return NULL. ** -** If an error occurs and pErrCtx==0 then return the Parse object with -** JsonParse.nErr non-zero. If the caller invokes this routine with -** pErrCtx==0 and it gets back a JsonParse with nErr!=0, then the caller -** is responsible for invoking jsonParseFree() on the returned value. -** But the caller may invoke jsonParseFree() *only* if pParse->nErr!=0. +** The returned pointer (if it is not NULL) is owned by the cache in +** most cases, not the caller. The caller does NOT need to invoke +** jsonParseFree(), in most cases. +** +** Except, if an error occurs and pErrCtx==0 then return the JsonParse +** object with JsonParse.nErr non-zero and the caller will own the JsonParse +** object. In that case, it will be the responsibility of the caller to +** invoke jsonParseFree(). To summarize: +** +** pErrCtx!=0 || p->nErr==0 ==> Return value p is owned by the +** cache. Call does not need to +** free it. +** +** pErrCtx==0 && p->nErr!=0 ==> Return value is owned by the caller +** and so the caller must free it. */ static JsonParse *jsonParseCached( - sqlite3_context *pCtx, - sqlite3_value **argv, - sqlite3_context *pErrCtx + sqlite3_context *pCtx, /* Context to use for cache search */ + sqlite3_value *pJson, /* Function param containing JSON text */ + sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ + int bUnedited /* No prior edits allowed */ ){ - const char *zJson = (const char*)sqlite3_value_text(argv[0]); - int nJson = sqlite3_value_bytes(argv[0]); + char *zJson = (char*)sqlite3_value_text(pJson); + int nJson = sqlite3_value_bytes(pJson); JsonParse *p; JsonParse *pMatch = 0; int iKey; int iMinKey = 0; u32 iMinHold = 0xffffffff; u32 iMaxHold = 0; + int bJsonRCStr; + if( zJson==0 ) return 0; for(iKey=0; iKeynJson==nJson - && memcmp(p->zJson,zJson,nJson)==0 + && (p->hasMod==0 || bUnedited==0) + && (p->zJson==zJson || memcmp(p->zJson,zJson,nJson)==0) ){ p->nErr = 0; + p->useMod = 0; + pMatch = p; + }else + if( pMatch==0 + && p->zAlt!=0 + && bUnedited==0 + && p->nAlt==nJson + && memcmp(p->zAlt, zJson, nJson)==0 + ){ + p->nErr = 0; + p->useMod = 1; pMatch = p; }else if( p->iHoldiHold; @@ -1698,28 +1915,44 @@ static JsonParse *jsonParseCached( } } if( pMatch ){ + /* The input JSON text was found in the cache. Use the preexisting + ** parse of this JSON */ pMatch->nErr = 0; pMatch->iHold = iMaxHold+1; + assert( pMatch->nJPRef>0 ); /* pMatch is owned by the cache */ return pMatch; } - p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); + + /* The input JSON was not found anywhere in the cache. We will need + ** to parse it ourselves and generate a new JsonParse object. + */ + bJsonRCStr = sqlite3ValueIsOfClass(pJson,(void(*)(void*))sqlite3RCStrUnref); + p = sqlite3_malloc64( sizeof(*p) + (bJsonRCStr ? 0 : nJson+1) ); if( p==0 ){ sqlite3_result_error_nomem(pCtx); return 0; } memset(p, 0, sizeof(*p)); - p->zJson = (char*)&p[1]; - memcpy((char*)p->zJson, zJson, nJson+1); - if( jsonParse(p, pErrCtx, p->zJson) ){ + if( bJsonRCStr ){ + p->zJson = sqlite3RCStrRef(zJson); + p->bJsonIsRCStr = 1; + }else{ + p->zJson = (char*)&p[1]; + memcpy(p->zJson, zJson, nJson+1); + } + p->nJPRef = 1; + if( jsonParse(p, pErrCtx) ){ if( pErrCtx==0 ){ p->nErr = 1; + assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */ return p; } - sqlite3_free(p); + jsonParseFree(p); return 0; } p->nJson = nJson; p->iHold = iMaxHold+1; + /* Transfer ownership of the new JsonParse to the cache */ sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, (void(*)(void*))jsonParseFree); return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); @@ -1770,9 +2003,26 @@ static JsonNode *jsonLookupStep( ){ u32 i, j, nKey; const char *zKey; - JsonNode *pRoot = &pParse->aNode[iRoot]; + JsonNode *pRoot; + if( pParse->oom ) return 0; + pRoot = &pParse->aNode[iRoot]; + while( (pRoot->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ + u32 idx = (u32)(pRoot - pParse->aNode); + i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( inNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==4 ); + assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ + pRoot = &pParse->aNode[i+1]; + iRoot = i+1; + break; + } + i = pParse->aNode[i].u.iPrev; + } + } if( zPath[0]==0 ) return pRoot; - if( pRoot->jnFlags & JNODE_REPLACE ) return 0; if( zPath[0]=='.' ){ if( pRoot->eType!=JSON_OBJECT ) return 0; zPath++; @@ -1806,14 +2056,16 @@ static JsonNode *jsonLookupStep( j += jsonNodeSize(&pRoot[j]); } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pRoot->eU==2 ); - iRoot += pRoot->u.iAppend; + iRoot = pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; j = 1; } if( pApnd ){ u32 iStart, iLabel; JsonNode *pNode; + assert( pParse->useMod ); iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); zPath += i; @@ -1822,7 +2074,7 @@ static JsonNode *jsonLookupStep( if( pNode ){ pRoot = &pParse->aNode[iRoot]; assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart - iRoot; + pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; VVA( pRoot->eU = 2 ); pParse->aNode[iLabel].jnFlags |= JNODE_RAW; @@ -1843,12 +2095,13 @@ static JsonNode *jsonLookupStep( if( pRoot->eType!=JSON_ARRAY ) return 0; for(;;){ while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; j += jsonNodeSize(&pBase[j]); } if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pBase->eU==2 ); - iBase += pBase->u.iAppend; + iBase = pBase->u.iAppend; pBase = &pParse->aNode[iBase]; j = 1; } @@ -1876,13 +2129,16 @@ static JsonNode *jsonLookupStep( zPath += j + 1; j = 1; for(;;){ - while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; + while( j<=pRoot->n + && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE)!=0 && pParse->useMod)) + ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i--; j += jsonNodeSize(&pRoot[j]); } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pRoot->eU==2 ); - iRoot += pRoot->u.iAppend; + iRoot = pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; j = 1; } @@ -1892,13 +2148,14 @@ static JsonNode *jsonLookupStep( if( i==0 && pApnd ){ u32 iStart; JsonNode *pNode; + assert( pParse->useMod ); iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); if( pParse->oom ) return 0; if( pNode ){ pRoot = &pParse->aNode[iRoot]; assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart - iRoot; + pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; VVA( pRoot->eU = 2 ); } @@ -1998,7 +2255,7 @@ static void jsonWrongNumArgs( char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", zFuncName); sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); + sqlite3_free(zMsg); } /* @@ -2025,47 +2282,90 @@ static void jsonRemoveAllNulls(JsonNode *pNode){ ** SQL functions used for testing and debugging ****************************************************************************/ +#if SQLITE_DEBUG +/* +** Print N node entries. +*/ +static void jsonDebugPrintNodeEntries( + JsonNode *aNode, /* First node entry to print */ + int N /* Number of node entries to print */ +){ + int i; + for(i=0; iaNode, p->nNode); +} +static void jsonDebugPrintNode(JsonNode *pNode){ + jsonDebugPrintNodeEntries(pNode, jsonNodeSize(pNode)); +} +#else + /* The usual case */ +# define jsonDebugPrintNode(X) +# define jsonDebugPrintParse(X) +#endif + #ifdef SQLITE_DEBUG /* -** The json_parse(JSON) function returns a string which describes -** a parse of the JSON provided. Or it returns NULL if JSON is not -** well-formed. +** SQL function: json_parse(JSON) +** +** Parse JSON using jsonParseCached(). Then print a dump of that +** parse on standard output. Return the mimified JSON result, just +** like the json() function. */ static void jsonParseFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv ){ - JsonString s; /* Output string - not real JSON */ - JsonParse x; /* The parse */ - u32 i; + JsonParse *p; /* The parse */ assert( argc==1 ); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonParseFindParents(&x); - jsonInit(&s, ctx); - for(i=0; inNode); + printf("nAlloc = %u\n", p->nAlloc); + printf("nJson = %d\n", p->nJson); + printf("nAlt = %d\n", p->nAlt); + printf("nErr = %u\n", p->nErr); + printf("oom = %u\n", p->oom); + printf("hasNonstd = %u\n", p->hasNonstd); + printf("useMod = %u\n", p->useMod); + printf("hasMod = %u\n", p->hasMod); + printf("nJPRef = %u\n", p->nJPRef); + printf("iSubst = %u\n", p->iSubst); + printf("iHold = %u\n", p->iHold); + jsonDebugPrintNodeEntries(p->aNode, p->nNode); + jsonReturnJson(p, p->aNode, ctx, 1); } /* @@ -2089,7 +2389,7 @@ static void jsonTest1Func( /* ** Implementation of the json_QUOTE(VALUE) function. Return a JSON value -** corresponding to the SQL value input. Mostly this means putting +** corresponding to the SQL value input. Mostly this means putting ** double-quotes around strings and returning the unquoted string "null" ** when given a NULL input. */ @@ -2136,7 +2436,7 @@ static void jsonArrayFunc( ** json_array_length(JSON) ** json_array_length(JSON, PATH) ** -** Return the number of elements in the top-level JSON array. +** Return the number of elements in the top-level JSON array. ** Return 0 if the input is not a well-formed JSON array. */ static void jsonArrayLengthFunc( @@ -2149,7 +2449,7 @@ static void jsonArrayLengthFunc( u32 i; JsonNode *pNode; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; assert( p->nNode ); if( argc==2 ){ @@ -2162,9 +2462,14 @@ static void jsonArrayLengthFunc( return; } if( pNode->eType==JSON_ARRAY ){ - assert( (pNode->jnFlags & JNODE_APPEND)==0 ); - for(i=1; i<=pNode->n; n++){ - i += jsonNodeSize(&pNode[i]); + while( 1 /*exit-by-break*/ ){ + for(i=1; i<=pNode->n; n++){ + i += jsonNodeSize(&pNode[i]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( p->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &p->aNode[pNode->u.iAppend]; } } sqlite3_result_int64(ctx, n); @@ -2211,7 +2516,7 @@ static void jsonExtractFunc( JsonString jx; if( argc<2 ) return; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; if( argc==2 ){ /* With a single PATH argument */ @@ -2244,15 +2549,15 @@ static void jsonExtractFunc( } if( pNode ){ if( flags & JSON_JSON ){ - jsonReturnJson(pNode, ctx, 0); + jsonReturnJson(p, pNode, ctx, 0); }else{ - jsonReturn(pNode, ctx, 0); + jsonReturn(p, pNode, ctx); sqlite3_result_subtype(ctx, 0); } } }else{ pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturn(pNode, ctx, 0); + if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx); } }else{ /* Two or more PATH arguments results in a JSON array with each @@ -2266,7 +2571,7 @@ static void jsonExtractFunc( if( p->nErr ) break; jsonAppendSeparator(&jx); if( pNode ){ - jsonRenderNode(pNode, &jx, 0); + jsonRenderNode(p, pNode, &jx); }else{ jsonAppendRawNZ(&jx, "null", 4); } @@ -2313,45 +2618,38 @@ static JsonNode *jsonMergePatch( assert( pTarget[j].eType==JSON_STRING ); assert( pTarget[j].jnFlags & JNODE_LABEL ); if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){ - if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ) break; if( pPatch[i+1].eType==JSON_NULL ){ pTarget[j+1].jnFlags |= JNODE_REMOVE; }else{ JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); if( pNew==0 ) return 0; - pTarget = &pParse->aNode[iTarget]; - if( pNew!=&pTarget[j+1] ){ - assert( pTarget[j+1].eU==0 - || pTarget[j+1].eU==1 - || pTarget[j+1].eU==2 ); - testcase( pTarget[j+1].eU==1 ); - testcase( pTarget[j+1].eU==2 ); - VVA( pTarget[j+1].eU = 5 ); - pTarget[j+1].u.pPatch = pNew; - pTarget[j+1].jnFlags |= JNODE_PATCH; + if( pNew!=&pParse->aNode[iTarget+j+1] ){ + jsonParseAddSubstNode(pParse, iTarget+j+1); + jsonParseAddNodeArray(pParse, pNew, jsonNodeSize(pNew)); } + pTarget = &pParse->aNode[iTarget]; } break; } } if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ - int iStart, iPatch; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + int iStart; + JsonNode *pApnd; + u32 nApnd; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + pApnd = &pPatch[i+1]; + if( pApnd->eType==JSON_OBJECT ) jsonRemoveAllNulls(pApnd); + nApnd = jsonNodeSize(pApnd); + jsonParseAddNodeArray(pParse, pApnd, jsonNodeSize(pApnd)); if( pParse->oom ) return 0; - jsonRemoveAllNulls(pPatch); - pTarget = &pParse->aNode[iTarget]; - assert( pParse->aNode[iRoot].eU==0 || pParse->aNode[iRoot].eU==2 ); - testcase( pParse->aNode[iRoot].eU==2 ); + pParse->aNode[iStart].n = 1+nApnd; pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; + pParse->aNode[iRoot].u.iAppend = iStart; VVA( pParse->aNode[iRoot].eU = 2 ); - pParse->aNode[iRoot].u.iAppend = iStart - iRoot; iRoot = iStart; - assert( pParse->aNode[iPatch].eU==0 ); - VVA( pParse->aNode[iPatch].eU = 5 ); - pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; - pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + pTarget = &pParse->aNode[iTarget]; } } return pTarget; @@ -2367,25 +2665,27 @@ static void jsonPatchFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The JSON that is being patched */ - JsonParse y; /* The patch */ + JsonParse *pX; /* The JSON that is being patched */ + JsonParse *pY; /* The patch */ JsonNode *pResult; /* The result of the merge */ UNUSED_PARAMETER(argc); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ - jsonParseReset(&x); - return; - } - pResult = jsonMergePatch(&x, 0, y.aNode); - assert( pResult!=0 || x.oom ); - if( pResult ){ - jsonReturnJson(pResult, ctx, 0); + pX = jsonParseCached(ctx, argv[0], ctx, 1); + if( pX==0 ) return; + pY = jsonParseCached(ctx, argv[1], ctx, 1); + if( pY==0 ) return; + pX->useMod = 1; + pX->hasMod = 1; + pY->useMod = 1; + pResult = jsonMergePatch(pX, 0, pY->aNode); + assert( pResult!=0 || pX->oom ); + if( pResult && pX->oom==0 ){ + jsonDebugPrintParse(pX); + jsonDebugPrintNode(pResult); + jsonReturnJson(pX, pResult, ctx, 0); }else{ sqlite3_result_error_nomem(ctx); } - jsonParseReset(&x); - jsonParseReset(&y); } @@ -2441,26 +2741,118 @@ static void jsonRemoveFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ + JsonParse *pParse; /* The parse */ JsonNode *pNode; const char *zPath; u32 i; if( argc<1 ) return; - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); + if( pParse==0 ) return; for(i=1; i<(u32)argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) goto remove_done; - pNode = jsonLookup(&x, zPath, 0, ctx); - if( x.nErr ) goto remove_done; - if( pNode ) pNode->jnFlags |= JNODE_REMOVE; + pNode = jsonLookup(pParse, zPath, 0, ctx); + if( pParse->nErr ) goto remove_done; + if( pNode ){ + pNode->jnFlags |= JNODE_REMOVE; + pParse->hasMod = 1; + pParse->useMod = 1; + } } - if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(x.aNode, ctx, 0); + if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){ + jsonReturnJson(pParse, pParse->aNode, ctx, 1); } remove_done: - jsonParseReset(&x); + jsonDebugPrintParse(p); +} + +/* +** Substitute the value at iNode with the pValue parameter. +*/ +static void jsonReplaceNode( + sqlite3_context *pCtx, + JsonParse *p, + int iNode, + sqlite3_value *pValue +){ + int idx = jsonParseAddSubstNode(p, iNode); + if( idx<=0 ){ + assert( p->oom ); + return; + } + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonParseAddNode(p, JSON_NULL, 0, 0); + break; + } + case SQLITE_FLOAT: { + char *z = sqlite3_mprintf("%!0.15g", sqlite3_value_double(pValue)); + int n; + if( z==0 ){ + p->oom = 1; + break; + } + n = sqlite3Strlen30(z); + jsonParseAddNode(p, JSON_REAL, n, z); + jsonParseAddCleanup(p, sqlite3_free, z); + break; + } + case SQLITE_INTEGER: { + char *z = sqlite3_mprintf("%lld", sqlite3_value_int64(pValue)); + int n; + if( z==0 ){ + p->oom = 1; + break; + } + n = sqlite3Strlen30(z); + jsonParseAddNode(p, JSON_INT, n, z); + jsonParseAddCleanup(p, sqlite3_free, z); + + break; + } + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + if( z==0 ){ + p->oom = 1; + break; + } + if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){ + char *zCopy = sqlite3DbStrDup(0, z); + int k; + if( zCopy ){ + jsonParseAddCleanup(p, sqlite3_free, zCopy); + }else{ + p->oom = 1; + sqlite3_result_error_nomem(pCtx); + } + k = jsonParseAddNode(p, JSON_STRING, n, zCopy); + assert( k>0 || p->oom ); + if( p->oom==0 ) p->aNode[k].jnFlags |= JNODE_RAW; + }else{ + JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1); + if( pPatch==0 ){ + p->oom = 1; + break; + } + jsonParseAddNodeArray(p, pPatch->aNode, pPatch->nNode); + /* The nodes copied out of pPatch and into p likely contain + ** u.zJContent pointers into pPatch->zJson. So preserve the + ** content of pPatch until p is destroyed. */ + assert( pPatch->nJPRef>=1 ); + pPatch->nJPRef++; + jsonParseAddCleanup(p, (void(*)(void*))jsonParseFree, pPatch); + } + break; + } + default: { + jsonParseAddNode(p, JSON_NULL, 0, 0); + sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); + p->nErr++; + break; + } + } } /* @@ -2474,7 +2866,7 @@ static void jsonReplaceFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ + JsonParse *pParse; /* The parse */ JsonNode *pNode; const char *zPath; u32 i; @@ -2484,28 +2876,20 @@ static void jsonReplaceFunc( jsonWrongNumArgs(ctx, "replace"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); + if( pParse==0 ) return; for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); - pNode = jsonLookup(&x, zPath, 0, ctx); - if( x.nErr ) goto replace_err; + pParse->useMod = 1; + pNode = jsonLookup(pParse, zPath, 0, ctx); + if( pParse->nErr ) goto replace_err; if( pNode ){ - assert( pNode->eU==0 || pNode->eU==1 || pNode->eU==4 ); - testcase( pNode->eU!=0 && pNode->eU!=1 ); - pNode->jnFlags |= (u8)JNODE_REPLACE; - VVA( pNode->eU = 4 ); - pNode->u.iReplace = i + 1; + jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - assert( x.aNode[0].eU==4 ); - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } + jsonReturnJson(pParse, pParse->aNode, ctx, 1); replace_err: - jsonParseReset(&x); + jsonDebugPrintParse(pParse); } @@ -2526,7 +2910,7 @@ static void jsonSetFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ + JsonParse *pParse; /* The parse */ JsonNode *pNode; const char *zPath; u32 i; @@ -2538,33 +2922,27 @@ static void jsonSetFunc( jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); + if( pParse==0 ) return; for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); bApnd = 0; - pNode = jsonLookup(&x, zPath, &bApnd, ctx); - if( x.oom ){ + pParse->useMod = 1; + pNode = jsonLookup(pParse, zPath, &bApnd, ctx); + if( pParse->oom ){ sqlite3_result_error_nomem(ctx); goto jsonSetDone; - }else if( x.nErr ){ + }else if( pParse->nErr ){ goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ - testcase( pNode->eU!=0 && pNode->eU!=1 ); - assert( pNode->eU!=3 && pNode->eU!=5 ); - VVA( pNode->eU = 4 ); - pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->u.iReplace = i + 1; + jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - assert( x.aNode[0].eU==4 ); - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } + jsonDebugPrintParse(pParse); + jsonReturnJson(pParse, pParse->aNode, ctx, 1); + jsonSetDone: - jsonParseReset(&x); + /* no cleanup required */; } /* @@ -2583,7 +2961,7 @@ static void jsonTypeFunc( const char *zPath; JsonNode *pNode; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; if( argc==2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); @@ -2610,12 +2988,12 @@ static void jsonValidFunc( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv, 0); + p = jsonParseCached(ctx, argv[0], 0, 0); if( p==0 || p->oom ){ sqlite3_result_error_nomem(ctx); sqlite3_free(p); }else{ - sqlite3_result_int(ctx, p->nErr==0 && p->hasNonstd==0); + sqlite3_result_int(ctx, p->nErr==0 && (p->hasNonstd==0 || p->useMod)); if( p->nErr ) jsonParseFree(p); } } @@ -2656,7 +3034,7 @@ static void jsonErrorFunc( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv, 0); + p = jsonParseCached(ctx, argv[0], 0, 0); if( p==0 || p->oom ){ sqlite3_result_error_nomem(ctx); sqlite3_free(p); @@ -2665,7 +3043,7 @@ static void jsonErrorFunc( }else{ int n = 1; u32 i; - const char *z = p->zJson; + const char *z = (const char*)sqlite3_value_text(argv[0]); for(i=0; iiErr && ALWAYS(z[i]); i++){ if( (z[i]&0xc0)!=0x80 ) n++; } @@ -2713,7 +3091,8 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ assert( pStr->bStatic ); }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic ? SQLITE_TRANSIENT : + (void(*)(void*))sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); @@ -2821,7 +3200,8 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ assert( pStr->bStatic ); }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic ? SQLITE_TRANSIENT : + (void(*)(void*))sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); @@ -2889,7 +3269,7 @@ static int jsonEachConnect( UNUSED_PARAMETER(argv); UNUSED_PARAMETER(argc); UNUSED_PARAMETER(pAux); - rc = sqlite3_declare_vtab(db, + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," "json HIDDEN,root HIDDEN)"); if( rc==SQLITE_OK ){ @@ -2932,7 +3312,6 @@ static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ /* Reset a JsonEachCursor back to its original state. Free any memory ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ - sqlite3_free(p->zJson); sqlite3_free(p->zRoot); jsonParseReset(&p->sParse); p->iRowid = 0; @@ -3070,7 +3449,7 @@ static int jsonEachColumn( case JEACH_KEY: { if( p->i==0 ) break; if( p->eType==JSON_OBJECT ){ - jsonReturn(pThis, ctx, 0); + jsonReturn(&p->sParse, pThis, ctx); }else if( p->eType==JSON_ARRAY ){ u32 iKey; if( p->bRecursive ){ @@ -3086,7 +3465,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(pThis, ctx, 0); + jsonReturn(&p->sParse, pThis, ctx); break; } case JEACH_TYPE: { @@ -3097,11 +3476,11 @@ static int jsonEachColumn( case JEACH_ATOM: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(pThis, ctx, 0); + jsonReturn(&p->sParse, pThis, ctx); break; } case JEACH_ID: { - sqlite3_result_int64(ctx, + sqlite3_result_int64(ctx, (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); break; } @@ -3201,8 +3580,8 @@ static int jsonEachBestIndex( idxMask |= iMask; } } - if( pIdxInfo->nOrderBy>0 - && pIdxInfo->aOrderBy[0].iColumn<0 + if( pIdxInfo->nOrderBy>0 + && pIdxInfo->aOrderBy[0].iColumn<0 && pIdxInfo->aOrderBy[0].desc==0 ){ pIdxInfo->orderByConsumed = 1; @@ -3252,11 +3631,19 @@ static int jsonEachFilter( if( idxNum==0 ) return SQLITE_OK; z = (const char*)sqlite3_value_text(argv[0]); if( z==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[0]); - p->zJson = sqlite3_malloc64( n+1 ); - if( p->zJson==0 ) return SQLITE_NOMEM; - memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson) ){ + memset(&p->sParse, 0, sizeof(p->sParse)); + p->sParse.nJPRef = 1; + if( sqlite3ValueIsOfClass(argv[0], (void(*)(void*))sqlite3RCStrUnref) ){ + p->sParse.zJson = sqlite3RCStrRef((char*)z); + }else{ + n = sqlite3_value_bytes(argv[0]); + p->sParse.zJson = sqlite3RCStrNew( n+1 ); + if( p->sParse.zJson==0 ) return SQLITE_NOMEM; + memcpy(p->sParse.zJson, z, (size_t)n+1); + } + p->sParse.bJsonIsRCStr = 1; + p->zJson = p->sParse.zJson; + if( jsonParse(&p->sParse, 0) ){ int rc = SQLITE_NOMEM; if( p->sParse.oom==0 ){ sqlite3_free(cur->pVtab->zErrMsg); @@ -3402,10 +3789,10 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_parse, 1, 0, jsonParseFunc), JFUNCTION(json_test1, 1, 0, jsonTest1Func), #endif - WAGGREGATE(json_group_array, 1, 0, 0, + WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), - WAGGREGATE(json_group_object, 2, 0, 0, + WAGGREGATE(json_group_object, 2, 0, 0, jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC) }; diff --git a/src/parse.y b/src/parse.y index 296ce6ffc1..6253b6f361 100644 --- a/src/parse.y +++ b/src/parse.y @@ -155,8 +155,8 @@ ecmd ::= SEMI. ecmd ::= cmdx SEMI. %ifndef SQLITE_OMIT_EXPLAIN ecmd ::= explain cmdx SEMI. {NEVER-REDUCE} -explain ::= EXPLAIN. { pParse->explain = 1; } -explain ::= EXPLAIN QUERY PLAN. { pParse->explain = 2; } +explain ::= EXPLAIN. { if( pParse->pReprepare==0 ) pParse->explain = 1; } +explain ::= EXPLAIN QUERY PLAN. { if( pParse->pReprepare==0 ) pParse->explain = 2; } %endif SQLITE_OMIT_EXPLAIN cmdx ::= cmd. { sqlite3FinishCoding(pParse); } diff --git a/src/prepare.c b/src/prepare.c index 39e8dcf655..9f843faa86 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -700,7 +700,12 @@ static int sqlite3Prepare( sParse.pOuterParse = db->pParse; db->pParse = &sParse; sParse.db = db; - sParse.pReprepare = pReprepare; + if( pReprepare ){ + sParse.pReprepare = pReprepare; + sParse.explain = sqlite3_stmt_isexplain((sqlite3_stmt*)pReprepare); + }else{ + assert( sParse.pReprepare==0 ); + } assert( ppStmt && *ppStmt==0 ); if( db->mallocFailed ){ sqlite3ErrorMsg(&sParse, "out of memory"); diff --git a/src/printf.c b/src/printf.c index 3fb1a322a0..87ad91f795 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1366,3 +1366,72 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){ sqlite3_str_vappendf(p, zFormat, ap); va_end(ap); } + + +/***************************************************************************** +** Reference counted string storage +*****************************************************************************/ + +/* +** Increase the reference count of the string by one. +** +** The input parameter is returned. +*/ +char *sqlite3RCStrRef(char *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + p->nRCRef++; + return z; +} + +/* +** Decrease the reference count by one. Free the string when the +** reference count reaches zero. +*/ +void sqlite3RCStrUnref(char *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + assert( p->nRCRef>0 ); + if( p->nRCRef>=2 ){ + p->nRCRef--; + }else{ + sqlite3_free(p); + } +} + +/* +** Create a new string that is capable of holding N bytes of text, not counting +** the zero byte at the end. The string is uninitialized. +** +** The reference count is initially 1. Call sqlite3RCStrUnref() to free the +** newly allocated string. +** +** This routine returns 0 on an OOM. +*/ +char *sqlite3RCStrNew(u64 N){ + RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 ); + if( p==0 ) return 0; + p->nRCRef = 1; + return (char*)&p[1]; +} + +/* +** Change the size of the string so that it is able to hold N bytes. +** The string might be reallocated, so return the new allocation. +*/ +char *sqlite3RCStrResize(char *z, u64 N){ + RCStr *p = (RCStr*)z; + RCStr *pNew; + assert( p!=0 ); + p--; + assert( p->nRCRef==1 ); + pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1); + if( pNew==0 ){ + sqlite3_free(p); + return 0; + }else{ + return (char*)&pNew[1]; + } +} diff --git a/src/select.c b/src/select.c index 087aa4907a..ce688663da 100644 --- a/src/select.c +++ b/src/select.c @@ -2101,13 +2101,6 @@ void sqlite3GenerateColumnNames( int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */ int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */ -#ifndef SQLITE_OMIT_EXPLAIN - /* If this is an EXPLAIN, skip this step */ - if( pParse->explain ){ - return; - } -#endif - if( pParse->colNamesSet ) return; /* Column names are determined by the left-most term of a compound select */ while( pSelect->pPrior ) pSelect = pSelect->pPrior; diff --git a/src/shell.c.in b/src/shell.c.in index 9165f21f71..1e65d0e9fd 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1607,6 +1607,7 @@ static ShellState shellState; #define MODE_Box 16 /* Unicode box-drawing characters */ #define MODE_Count 17 /* Output only a count of the rows of output */ #define MODE_Off 18 /* No query output shown */ +#define MODE_ScanExp 19 /* Like MODE_Explain, but for ".scanstats vm" */ static const char *modeDescr[] = { "line", @@ -2520,38 +2521,58 @@ static int shell_callback( } break; } + case MODE_ScanExp: case MODE_Explain: { - static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; - if( nArg>ArraySize(aExplainWidth) ){ - nArg = ArraySize(aExplainWidth); + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4, 6, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = ArraySize(aExplainWidth); + int iIndent = 1; + + if( p->cMode==MODE_ScanExp ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = ArraySize(aScanExpWidth); + iIndent = 3; } + if( nArg>nWidth ) nArg = nWidth; + + /* If this is the first row seen, print out the headers */ if( p->cnt++==0 ){ for(i=0; iout, w, azCol[i]); + utf8_width_print(p->out, aWidth[i], azCol[ aMap[i] ]); fputs(i==nArg-1 ? "\n" : " ", p->out); } for(i=0; iout, w); + print_dashes(p->out, aWidth[i]); fputs(i==nArg-1 ? "\n" : " ", p->out); } } + + /* If there is no data, exit early. */ if( azArg==0 ) break; + for(i=0; iw ){ - w = strlenChar(azArg[i]); + if( zVal && strlenChar(zVal)>w ){ + w = strlenChar(zVal); + zSep = " "; } - if( i==1 && p->aiIndent && p->pStmt ){ + if( i==iIndent && p->aiIndent && p->pStmt ){ if( p->iIndentnIndent ){ utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); } p->iIndent++; } - utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue); - fputs(i==nArg-1 ? "\n" : " ", p->out); + utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); + fputs(i==nArg-1 ? "\n" : zSep, p->out); } break; } @@ -3332,17 +3353,11 @@ static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ } #endif -/* -** Display scan stats. -*/ -static void display_scanstats( +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static void display_explain_scanstats( sqlite3 *db, /* Database to query */ ShellState *pArg /* Pointer to ShellState */ ){ -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(pArg); -#else static const int f = SQLITE_SCANSTAT_COMPLEX; sqlite3_stmt *p = pArg->pStmt; int ii = 0; @@ -3414,8 +3429,9 @@ static void display_scanstats( } eqp_render(pArg, nTotal); -#endif } +#endif + /* ** Parameter azArray points to a zero-terminated array of strings. zStr @@ -3453,8 +3469,6 @@ static int str_in_array(const char *zStr, const char **azArray){ ** and "Goto" by 2 spaces. */ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ - const char *zSql; /* The text of the SQL statement */ - const char *z; /* Used to check if this is an EXPLAIN */ int *abYield = 0; /* True if op is an OP_Yield */ int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ int iOp; /* Index of operation in p->aiIndent[] */ @@ -3465,65 +3479,45 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; - /* Try to figure out if this is really an EXPLAIN statement. If this - ** cannot be verified, return early. */ - if( sqlite3_column_count(pSql)!=8 ){ - p->cMode = p->mode; - return; - } - zSql = sqlite3_sql(pSql); - if( zSql==0 ) return; - for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++); - if( sqlite3_strnicmp(z, "explain", 7) ){ - p->cMode = p->mode; - return; - } + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(pSql)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) ); for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ int i; int iAddr = sqlite3_column_int(pSql, 0); const char *zOp = (const char*)sqlite3_column_text(pSql, 1); - - /* Set p2 to the P2 field of the current opcode. Then, assuming that - ** p2 is an instruction address, set variable p2op to the index of that - ** instruction in the aiIndent[] array. p2 and p2op may be different if - ** the current instruction is part of a sub-program generated by an - ** SQL trigger or foreign key. */ + int p1 = sqlite3_column_int(pSql, 2); int p2 = sqlite3_column_int(pSql, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ int p2op = (p2 + (iOp-iAddr)); /* Grow the p->aiIndent array as required */ if( iOp>=nAlloc ){ - if( iOp==0 ){ - /* Do further verification that this is explain output. Abort if - ** it is not */ - static const char *explainCols[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment" }; - int jj; - for(jj=0; jjcMode = p->mode; - sqlite3_reset(pSql); - return; - } - } - } nAlloc += 100; p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); shell_check_oom(p->aiIndent); abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); shell_check_oom(abYield); } + abYield[iOp] = str_in_array(zOp, azYield); p->aiIndent[iOp] = 0; p->nIndent = iOp+1; - if( str_in_array(zOp, azNext) && p2op>0 ){ for(i=p2op; iaiIndent[i] += 2; } - if( str_in_array(zOp, azGoto) && p2opnIndent - && (abYield[p2op] || sqlite3_column_int(pSql, 2)) - ){ + if( str_in_array(zOp, azGoto) && p2opaiIndent[i] += 2; } } @@ -3543,6 +3537,48 @@ static void explain_data_delete(ShellState *p){ p->iIndent = 0; } +static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); + +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pArg); +#else + if( pArg->scanstatsOn==3 ){ + const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " round(ncycle*100.0 / (sum(ncycle) OVER ()), 2)||'%' AS cycles" + " FROM bytecode(?)"; + + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSave = pArg->pStmt; + pArg->pStmt = pStmt; + sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); + + pArg->cnt = 0; + pArg->cMode = MODE_ScanExp; + explain_data_prepare(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); + explain_data_delete(pArg); + + sqlite3_finalize(pStmt); + pArg->pStmt = pSave; + } + }else{ + display_explain_scanstats(db, pArg); + } +#endif +} + /* ** Disable and restore .wheretrace and .treetrace/.selecttrace settings. */ @@ -4332,16 +4368,15 @@ static int shell_exec( /* Show the EXPLAIN QUERY PLAN if .eqp is on */ if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){ sqlite3_stmt *pExplain; - char *zEQP; int triggerEQP = 0; disable_debug_trace_modes(); sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP); if( pArg->autoEQP>=AUTOEQP_trigger ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0); } - zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql); - shell_check_oom(zEQP); - rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + pExplain = pStmt; + sqlite3_reset(pExplain); + rc = sqlite3_stmt_explain(pExplain, 2); if( rc==SQLITE_OK ){ while( sqlite3_step(pExplain)==SQLITE_ROW ){ const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); @@ -4353,36 +4388,31 @@ static int shell_exec( } eqp_render(pArg, 0); } - sqlite3_finalize(pExplain); - sqlite3_free(zEQP); if( pArg->autoEQP>=AUTOEQP_full ){ /* Also do an EXPLAIN for ".eqp full" mode */ - zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql); - shell_check_oom(zEQP); - rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + sqlite3_reset(pExplain); + rc = sqlite3_stmt_explain(pExplain, 1); if( rc==SQLITE_OK ){ pArg->cMode = MODE_Explain; + assert( sqlite3_stmt_isexplain(pExplain)==1 ); explain_data_prepare(pArg, pExplain); exec_prepared_stmt(pArg, pExplain); explain_data_delete(pArg); } - sqlite3_finalize(pExplain); - sqlite3_free(zEQP); } if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0); - /* Reprepare pStmt before reactivating trace modes */ - sqlite3_finalize(pStmt); - sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( pArg ) pArg->pStmt = pStmt; } + sqlite3_reset(pStmt); + sqlite3_stmt_explain(pStmt, 0); restore_debug_trace_modes(); } if( pArg ){ + int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1); pArg->cMode = pArg->mode; if( pArg->autoExplain ){ - if( sqlite3_stmt_isexplain(pStmt)==1 ){ + if( bIsExplain ){ pArg->cMode = MODE_Explain; } if( sqlite3_stmt_isexplain(pStmt)==2 ){ @@ -4392,7 +4422,7 @@ static int shell_exec( /* If the shell is currently in ".explain" mode, gather the extra ** data required to add indents to the output.*/ - if( pArg->cMode==MODE_Explain ){ + if( pArg->cMode==MODE_Explain && bIsExplain ){ explain_data_prepare(pArg, pStmt); } } @@ -9933,6 +9963,9 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ + if( cli_strcmp(azArg[1], "vm")==0 ){ + p->scanstatsOn = 3; + }else if( cli_strcmp(azArg[1], "est")==0 ){ p->scanstatsOn = 2; }else{ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d3f44ad5d1..636a43aa9f 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4421,6 +4421,41 @@ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); */ int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN +** setting for prepared statement S. If E is zero, then S becomes +** a normal prepared statement. If E is 1, then S behaves as if +** its SQL text began with "EXPLAIN". If E is 2, then S behaves as if +** its SQL text began with "EXPLAIN QUERY PLAN". +** +** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared. +** SQLite tries to avoid a reprepare, but a reprepare might be necessary +** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode. +** +** Because of the potential need to reprepare, a call to +** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be +** reprepared because it was created using sqlite3_prepare() instead of +** the newer sqlite_prepare_v2() or sqlite3_prepare_v3() interfaces and +** hence has no saved SQL text with which to reprepare. +** +** Changing the explain setting for a prepared statement does not change +** the original SQL text for the statement. Hence, if the SQL text originally +** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0) +** is called to convert the statement into an ordinary statement, the EXPLAIN +** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S) +** output, even though the statement now acts like a normal SQL statement. +** +** This routine returns SQLITE_OK if the explain mode is successfully +** changed, or an error code if the explain mode could not be changed. +** The explain mode cannot be changed while a statement is active. +** Hence, it is good practice to call [sqlite3_reset(S)] +** immediately prior to calling sqlite3_stmt_explain(S,E). +*/ +int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode); + /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset ** METHOD: sqlite3_stmt diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 27c479481a..4a74a9aaf6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1282,6 +1282,7 @@ typedef struct Parse Parse; typedef struct ParseCleanup ParseCleanup; typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; +typedef struct RCStr RCStr; typedef struct RenameToken RenameToken; typedef struct Returning Returning; typedef struct RowSet RowSet; @@ -4082,6 +4083,25 @@ struct sqlite3_str { #define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) +/* +** The following object is the header for an "RCStr" or "reference-counted +** string". An RCStr is passed around and used like any other char* +** that has been dynamically allocated. The important interface +** differences: +** +** 1. RCStr strings are reference counted. They are deallocated +** when the reference count reaches zero. +** +** 2. Use sqlite3RCStrUnref() to free an RCStr string rather than +** sqlite3_free() +** +** 3. Make a (read-only) copy of a read-only RCStr string using +** sqlite3RCStrRef(). +*/ +struct RCStr { + u64 nRCRef; /* Number of references */ + /* Total structure size should be a multiple of 8 bytes for alignment */ +}; /* ** A pointer to this structure is used to communicate information @@ -5203,6 +5223,7 @@ void sqlite3FileSuffix3(const char*, char*); u8 sqlite3GetBoolean(const char *z,u8); const void *sqlite3ValueText(sqlite3_value*, u8); +int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*)); int sqlite3ValueBytes(sqlite3_value*, u8); void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); @@ -5310,6 +5331,11 @@ void sqlite3OomClear(sqlite3*); int sqlite3ApiExit(sqlite3 *db, int); int sqlite3OpenTempDatabase(Parse *); +char *sqlite3RCStrRef(char*); +void sqlite3RCStrUnref(char*); +char *sqlite3RCStrNew(u64); +char *sqlite3RCStrResize(char*,u64); + void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); int sqlite3StrAccumEnlarge(StrAccum*, i64); char *sqlite3StrAccumFinish(StrAccum*); diff --git a/src/test1.c b/src/test1.c index 832b898585..307cc96f28 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2922,6 +2922,34 @@ static int SQLITE_TCLAPI test_stmt_isexplain( return TCL_OK; } +/* +** Usage: sqlite3_stmt_explain STMT INT +** +** Set the explain to normal (0), EXPLAIN (1) or EXPLAIN QUERY PLAN (2). +*/ +static int SQLITE_TCLAPI test_stmt_explain( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; + int eMode = 0; + int rc; + + if( objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " STMT INT", 0); + return TCL_ERROR; + } + + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &eMode) ) return TCL_ERROR; + rc = sqlite3_stmt_explain(pStmt, eMode); + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_OK; +} + /* ** Usage: sqlite3_stmt_busy STMT ** @@ -9026,6 +9054,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_next_stmt", test_next_stmt ,0 }, { "sqlite3_stmt_readonly", test_stmt_readonly ,0 }, { "sqlite3_stmt_isexplain", test_stmt_isexplain,0 }, + { "sqlite3_stmt_explain", test_stmt_explain ,0 }, { "sqlite3_stmt_busy", test_stmt_busy ,0 }, { "uses_stmt_journal", uses_stmt_journal ,0 }, diff --git a/src/vdbe.c b/src/vdbe.c index 9d210e70a0..12df44d89a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -695,6 +695,83 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ return h; } + +/* +** For OP_Column, factor out the case where content is loaded from +** overflow pages, so that the code to implement this case is separate +** the common case where all content fits on the page. Factoring out +** the code reduces register pressure and helps the common case +** to run faster. +*/ +static SQLITE_NOINLINE int vdbeColumnFromOverflow( + VdbeCursor *pC, /* The BTree cursor from which we are reading */ + int iCol, /* The column to read */ + int t, /* The serial-type code for the column value */ + i64 iOffset, /* Offset to the start of the content value */ + Mem *pDest /* Store the value into this register. */ +){ + int rc; + sqlite3 *db = pDest->db; + int encoding = pDest->enc; + int len = sqlite3VdbeSerialTypeLen(t); + assert( pC->eCurType==CURTYPE_BTREE ); + if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) return SQLITE_TOOBIG; + if( len > 4000 ){ + /* Cache large column values that are on overflow pages using + ** an RCStr (reference counted string) so that if they are reloaded, + ** that do not have to be copied a second time. The overhead of + ** creating and managing the cache is such that this is only + ** profitable for larger TEXT and BLOB values. + */ + VdbeTxtBlbCache *pCache; + char *pBuf; + if( pC->colCache==0 ){ + pC->pCache = sqlite3DbMallocZero(db, sizeof(VdbeTxtBlbCache) ); + if( pC->pCache==0 ) return SQLITE_NOMEM; + pC->colCache = 1; + } + pCache = pC->pCache; + if( pCache->pCValue==0 + || pCache->iCol!=iCol + || pCache->iOffset!=sqlite3BtreeOffset(pC->uc.pCursor) + ){ + if( pCache->pCValue ) sqlite3RCStrUnref(pCache->pCValue); + pBuf = pCache->pCValue = sqlite3RCStrNew( len+3 ); + if( pBuf==0 ) return SQLITE_NOMEM; + rc = sqlite3BtreePayload(pC->uc.pCursor, iOffset, len, pBuf); + if( rc ) return rc; + pBuf[len] = 0; + pBuf[len+1] = 0; + pBuf[len+2] = 0; + pCache->iCol = iCol; + pCache->iOffset = sqlite3BtreeOffset(pC->uc.pCursor); + }else{ + pBuf = pCache->pCValue; + } + assert( t>=12 ); + sqlite3RCStrRef(pBuf); + if( t&1 ){ + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding, + (void(*)(void*))sqlite3RCStrUnref); + pDest->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0, + (void(*)(void*))sqlite3RCStrUnref); + } + }else{ + rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest); + if( rc ) return rc; + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + if( (t&1)!=0 && encoding==SQLITE_UTF8 ){ + pDest->z[len] = 0; + pDest->flags |= MEM_Term; + } + } + pDest->flags &= ~MEM_Ephem; + return rc; +} + + /* ** Return the symbolic name for the data type of a pMem */ @@ -3061,6 +3138,7 @@ op_column_restart: }else{ u8 p5; pDest->enc = encoding; + assert( pDest->db==db ); /* This branch happens only when content is on overflow pages */ if( ((p5 = (pOp->p5 & OPFLAG_BYTELENARG))!=0 && (p5==OPFLAG_TYPEOFARG @@ -3084,14 +3162,12 @@ op_column_restart: */ sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest); }else{ - if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big; - rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest); - if( rc!=SQLITE_OK ) goto abort_due_to_error; - sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); - if( (t&1)!=0 && encoding==SQLITE_UTF8 ){ - pDest->flags |= MEM_Term; + rc = vdbeColumnFromOverflow(pC, p2, t, aOffset[p2], pDest); + if( rc ){ + if( rc==SQLITE_NOMEM ) goto no_mem; + if( rc==SQLITE_TOOBIG ) goto too_big; + goto abort_due_to_error; } - pDest->flags &= ~MEM_Ephem; } } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 3a5b961a25..f424559929 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -56,6 +56,9 @@ typedef struct VdbeSorter VdbeSorter; /* Elements of the linked list at Vdbe.pAuxData */ typedef struct AuxData AuxData; +/* A cache of large TEXT or BLOB values in a VdbeCursor */ +typedef struct VdbeTxtBlbCache VdbeTxtBlbCache; + /* Types of VDBE cursors */ #define CURTYPE_BTREE 0 #define CURTYPE_SORTER 1 @@ -87,6 +90,7 @@ struct VdbeCursor { Bool useRandomRowid:1; /* Generate new record numbers semi-randomly */ Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ Bool noReuse:1; /* OpenEphemeral may not reuse this cursor */ + Bool colCache:1; /* pCache pointer is initialized and non-NULL */ u16 seekHit; /* See the OP_SeekHit and OP_IfNoHope opcodes */ union { /* pBtx for isEphermeral. pAltMap otherwise */ Btree *pBtx; /* Separate file holding temporary table */ @@ -127,6 +131,7 @@ struct VdbeCursor { #ifdef SQLITE_ENABLE_COLUMN_USED_MASK u64 maskUsed; /* Mask of columns used by this cursor */ #endif + VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */ /* 2*nField extra array elements allocated for aType[], beyond the one ** static element declared in the structure. nField total array slots for @@ -139,12 +144,23 @@ struct VdbeCursor { #define IsNullCursor(P) \ ((P)->eCurType==CURTYPE_PSEUDO && (P)->nullRow && (P)->seekResult==0) - /* ** A value for VdbeCursor.cacheStatus that means the cache is always invalid. */ #define CACHE_STALE 0 +/* +** Large TEXT or BLOB values can be slow to load, so we want to avoid +** loading them more than once. For that reason, large TEXT and BLOB values +** can be stored in a cache defined by this object, and attached to the +** VdbeCursor using the pCache field. +*/ +struct VdbeTxtBlbCache { + char *pCValue; /* A RCStr buffer to hold the value */ + i64 iOffset; /* File offset of the row being cached */ + int iCol; /* Column for which the cache is valid */ +}; + /* ** When a sub-program is executed (OP_Program), a structure of this type ** is allocated to store the current value of the program counter, as @@ -465,16 +481,18 @@ struct Vdbe { u32 nWrite; /* Number of write operations that have occurred */ #endif u16 nResColumn; /* Number of columns in one row of the result set */ + u16 nResAlloc; /* Column slots allocated to aColName[] */ u8 errorAction; /* Recovery action to do in case of an error */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 eVdbeState; /* On of the VDBE_*_STATE values */ bft expired:2; /* 1: recompile VM immediately 2: when convenient */ - bft explain:2; /* True if EXPLAIN present on SQL command */ + bft explain:2; /* 0: normal, 1: EXPLAIN, 2: EXPLAIN QUERY PLAN */ bft changeCntOn:1; /* True to update the change-counter */ bft usesStmtJournal:1; /* True if uses a statement journal */ bft readOnly:1; /* True for statements that do not write */ bft bIsReader:1; /* True for statements that read */ + bft haveEqpOps:1; /* Bytecode supports EXPLAIN QUERY PLAN */ yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ yDbMask lockMask; /* Subset of btreeMask that requires a lock */ u32 aCounter[9]; /* Counters used by sqlite3_stmt_status() */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 92bf38160f..cb631413c4 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1127,7 +1127,8 @@ int sqlite3_aggregate_count(sqlite3_context *p){ */ int sqlite3_column_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; - return pVm ? pVm->nResColumn : 0; + if( pVm==0 ) return 0; + return pVm->nResColumn; } /* @@ -1300,6 +1301,32 @@ int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ return iType; } +/* +** Column names appropriate for EXPLAIN or EXPLAIN QUERY PLAN. +*/ +static const char * const azExplainColNames8[] = { + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", /* EXPLAIN */ + "id", "parent", "notused", "detail" /* EQP */ +}; +static const u16 azExplainColNames16data[] = { + /* 0 */ 'a', 'd', 'd', 'r', 0, + /* 5 */ 'o', 'p', 'c', 'o', 'd', 'e', 0, + /* 12 */ 'p', '1', 0, + /* 15 */ 'p', '2', 0, + /* 18 */ 'p', '3', 0, + /* 21 */ 'p', '4', 0, + /* 24 */ 'p', '5', 0, + /* 27 */ 'c', 'o', 'm', 'm', 'e', 'n', 't', 0, + /* 35 */ 'i', 'd', 0, + /* 38 */ 'p', 'a', 'r', 'e', 'n', 't', 0, + /* 45 */ 'n', 'o', 't', 'u', 's', 'e', 'd', 0, + /* 53 */ 'd', 'e', 't', 'a', 'i', 'l', 0 +}; +static const u8 iExplainColNames16[] = { + 0, 5, 12, 15, 18, 21, 24, 27, + 35, 38, 45, 53 +}; + /* ** Convert the N-th element of pStmt->pColName[] into a string using ** xFunc() then return that string. If N is out of range, return 0. @@ -1332,15 +1359,29 @@ static const void *columnName( return 0; } #endif + if( N<0 ) return 0; ret = 0; p = (Vdbe *)pStmt; db = p->db; assert( db!=0 ); - n = sqlite3_column_count(pStmt); - if( N=0 ){ + sqlite3_mutex_enter(db->mutex); + + if( p->explain ){ + if( useType>0 ) goto columnName_end; + n = p->explain==1 ? 8 : 4; + if( N>=n ) goto columnName_end; + if( useUtf16 ){ + int i = iExplainColNames16[N + 8*p->explain - 8]; + ret = (void*)&azExplainColNames16data[i]; + }else{ + ret = (void*)azExplainColNames8[N + 8*p->explain - 8]; + } + goto columnName_end; + } + n = p->nResColumn; + if( NmallocFailed; N += useType*n; - sqlite3_mutex_enter(db->mutex); #ifndef SQLITE_OMIT_UTF16 if( useUtf16 ){ ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); @@ -1357,8 +1398,9 @@ static const void *columnName( sqlite3OomClear(db); ret = 0; } - sqlite3_mutex_leave(db->mutex); } +columnName_end: + sqlite3_mutex_leave(db->mutex); return ret; } @@ -1815,6 +1857,35 @@ int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->explain : 0; } +/* +** Set the explain mode for a statement. +*/ +int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode){ + Vdbe *v = (Vdbe*)pStmt; + int rc; + if( v->eVdbeState!=VDBE_READY_STATE ) return SQLITE_BUSY; + if( v->explain==eMode ) return SQLITE_OK; + if( v->zSql==0 || eMode<0 || eMode>2 ) return SQLITE_ERROR; + sqlite3_mutex_enter(v->db->mutex); + if( v->nMem>=10 && (eMode!=2 || v->haveEqpOps) ){ + /* No reprepare necessary */ + v->explain = eMode; + rc = SQLITE_OK; + }else{ + int haveEqpOps = v->explain==2 || v->haveEqpOps; + v->explain = eMode; + rc = sqlite3Reprepare(v); + v->haveEqpOps = haveEqpOps!=0; + } + if( v->explain ){ + v->nResColumn = 12 - 4*v->explain; + }else{ + v->nResColumn = v->nResAlloc; + } + sqlite3_mutex_leave(v->db->mutex); + return rc; +} + /* ** Return true if the prepared statement is in need of being reset. */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 096a13e0e4..c97b6be4d1 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1498,7 +1498,6 @@ void sqlite3VdbeReleaseRegisters( } #endif /* SQLITE_DEBUG */ - /* ** Change the value of the P4 operand for a specific instruction. ** This routine is useful when a large program is loaded from a @@ -2418,8 +2417,8 @@ int sqlite3VdbeList( sqlite3VdbeMemSetInt64(pMem, pOp->p1); sqlite3VdbeMemSetInt64(pMem+1, pOp->p2); sqlite3VdbeMemSetInt64(pMem+2, pOp->p3); - sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free); - p->nResColumn = 4; + sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free); + assert( p->nResColumn==4 ); }else{ sqlite3VdbeMemSetInt64(pMem+0, i); sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode), @@ -2438,7 +2437,7 @@ int sqlite3VdbeList( sqlite3VdbeMemSetNull(pMem+7); #endif sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free); - p->nResColumn = 8; + assert( p->nResColumn==8 ); } p->pResultRow = pMem; if( db->mallocFailed ){ @@ -2652,26 +2651,9 @@ void sqlite3VdbeMakeReady( resolveP2Values(p, &nArg); p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); if( pParse->explain ){ - static const char * const azColName[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", - "id", "parent", "notused", "detail" - }; - int iFirst, mx, i; if( nMem<10 ) nMem = 10; p->explain = pParse->explain; - if( pParse->explain==2 ){ - sqlite3VdbeSetNumCols(p, 4); - iFirst = 8; - mx = 12; - }else{ - sqlite3VdbeSetNumCols(p, 8); - iFirst = 0; - mx = 8; - } - for(i=iFirst; inResColumn = 12 - 4*p->explain; } p->expired = 0; @@ -2723,7 +2705,23 @@ void sqlite3VdbeMakeReady( void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ if( pCx ) sqlite3VdbeFreeCursorNN(p,pCx); } +static SQLITE_NOINLINE void freeCursorWithCache(Vdbe *p, VdbeCursor *pCx){ + VdbeTxtBlbCache *pCache = pCx->pCache; + assert( pCx->colCache ); + pCx->colCache = 0; + pCx->pCache = 0; + if( pCache->pCValue ){ + sqlite3RCStrUnref(pCache->pCValue); + pCache->pCValue = 0; + } + sqlite3DbFree(p->db, pCache); + sqlite3VdbeFreeCursorNN(p, pCx); +} void sqlite3VdbeFreeCursorNN(Vdbe *p, VdbeCursor *pCx){ + if( pCx->colCache ){ + freeCursorWithCache(p, pCx); + return; + } switch( pCx->eCurType ){ case CURTYPE_SORTER: { sqlite3VdbeSorterClose(p->db, pCx); @@ -2824,12 +2822,12 @@ void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ int n; sqlite3 *db = p->db; - if( p->nResColumn ){ - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + if( p->nResAlloc ){ + releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N); sqlite3DbFree(db, p->aColName); } n = nResColumn*COLNAME_N; - p->nResColumn = (u16)nResColumn; + p->nResColumn = p->nResAlloc = (u16)nResColumn; p->aColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n ); if( p->aColName==0 ) return; initMemArray(p->aColName, n, db, MEM_Null); @@ -2854,14 +2852,14 @@ int sqlite3VdbeSetColName( ){ int rc; Mem *pColName; - assert( idxnResColumn ); + assert( idxnResAlloc ); assert( vardb->mallocFailed ){ assert( !zName || xDel!=SQLITE_DYNAMIC ); return SQLITE_NOMEM_BKPT; } assert( p->aColName!=0 ); - pColName = &(p->aColName[idx+var*p->nResColumn]); + pColName = &(p->aColName[idx+var*p->nResAlloc]); rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; @@ -3705,7 +3703,7 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ assert( db!=0 ); assert( p->db==0 || p->db==db ); if( p->aColName ){ - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N); sqlite3DbNNFreeNN(db, p->aColName); } for(pSub=p->pProgram; pSub; pSub=pNext){ diff --git a/src/vdbemem.c b/src/vdbemem.c index b5a794ae8f..07fb8feee8 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -333,6 +333,11 @@ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ pMem->flags |= MEM_Term; return; } + if( pMem->xDel==(void(*)(void*))sqlite3RCStrUnref ){ + /* Blindly assume that all RCStr objects are zero-terminated */ + pMem->flags |= MEM_Term; + return; + } }else if( pMem->szMalloc>0 && pMem->szMalloc >= pMem->n+1 ){ pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; @@ -1363,6 +1368,24 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return valueToText(pVal, enc); } +/* Return true if sqlit3_value object pVal is a string or blob value +** that uses the destructor specified in the second argument. +** +** TODO: Maybe someday promote this interface into a published API so +** that third-party extensions can get access to it? +*/ +int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){ + if( ALWAYS(pVal!=0) + && ALWAYS((pVal->flags & (MEM_Str|MEM_Blob))!=0) + && (pVal->flags & MEM_Dyn)!=0 + && pVal->xDel==xFree + ){ + return 1; + }else{ + return 0; + } +} + /* ** Create a new sqlite3_value object. */ diff --git a/src/vdbevtab.c b/src/vdbevtab.c index 6557d8cb01..59030e0e12 100644 --- a/src/vdbevtab.c +++ b/src/vdbevtab.c @@ -69,6 +69,8 @@ static int bytecodevtabConnect( "p5 INT," "comment TEXT," "subprog TEXT," + "nexec INT," + "ncycle INT," "stmt HIDDEN" ");", @@ -231,7 +233,7 @@ static int bytecodevtabColumn( } } } - i += 10; + i += 20; } } switch( i ){ @@ -281,16 +283,31 @@ static int bytecodevtabColumn( } break; } - case 10: /* tables_used.type */ + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + case 9: /* nexec */ + sqlite3_result_int(ctx, pOp->nExec); + break; + case 10: /* ncycle */ + sqlite3_result_int(ctx, pOp->nCycle); + break; +#else + case 9: /* nexec */ + case 10: /* ncycle */ + sqlite3_result_int(ctx, 0); + break; +#endif + + case 20: /* tables_used.type */ sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC); break; - case 11: /* tables_used.schema */ + case 21: /* tables_used.schema */ sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC); break; - case 12: /* tables_used.name */ + case 22: /* tables_used.name */ sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC); break; - case 13: /* tables_used.wr */ + case 23: /* tables_used.wr */ sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite); break; } @@ -364,7 +381,7 @@ static int bytecodevtabBestIndex( int rc = SQLITE_CONSTRAINT; struct sqlite3_index_constraint *p; bytecodevtab *pVTab = (bytecodevtab*)tab; - int iBaseCol = pVTab->bTablesUsed ? 4 : 8; + int iBaseCol = pVTab->bTablesUsed ? 4 : 10; pIdxInfo->estimatedCost = (double)100; pIdxInfo->estimatedRows = 100; pIdxInfo->idxNum = 0; diff --git a/test/fuzzdata6.db b/test/fuzzdata6.db index b1424c21e4..67076d0026 100644 Binary files a/test/fuzzdata6.db and b/test/fuzzdata6.db differ diff --git a/test/json/README.md b/test/json/README.md index a100287b05..55044ffca7 100644 --- a/test/json/README.md +++ b/test/json/README.md @@ -1,5 +1,6 @@ The files in this subdirectory are used to help measure the performance -of the SQLite JSON parser. +of the SQLite JSON functions, especially in relation to handling large +JSON inputs. # 1.0 Prerequisites @@ -7,6 +8,8 @@ of the SQLite JSON parser. 2. Fossil + 3. tclsh + # 2.0 Setup 1. Run: "`tclsh json-generator.tcl | sqlite3 json100mb.db`" to create @@ -14,16 +17,17 @@ of the SQLite JSON parser. file lands in the directory from which you will run tests, not in the test/json subdirectory of the source tree. - 2. Build the baseline sqlite3.c file. ("`make sqlite3.c`") + 2. Build the baseline sqlite3.c file with sqlite3.h and shell.c. + ("`CFLAGS='-Os -g' make -e clean sqlite3.c`") - 3. Run "`sh json-speed-check-1.sh trunk`". This creates the baseline + 3. Run "`sh json-speed-check.sh trunk`". This creates the baseline profile in "jout-trunk.txt". # 3.0 Testing - 1. Build the sqlite3.c to be tested. + 1. Build the sqlite3.c (with sqlite3.h and shell.c) to be tested. - 2. Run "`sh json-speed-check-1.sh x1`". The profile output will appear + 2. Run "`sh json-speed-check.sh x1`". The profile output will appear in jout-x1.txt. Substitute any label you want in place of "x1". 3. Run the script shown below in the CLI. diff --git a/test/json/json-q1.txt b/test/json/json-q1.txt index 0395f0c061..d122a2d826 100644 --- a/test/json/json-q1.txt +++ b/test/json/json-q1.txt @@ -2,3 +2,23 @@ .timer on .param set $label 'q87' SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL; + +CREATE TEMP TABLE t2(x JSON TEXT); +WITH RECURSIVE + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<25000), + array1(y) AS ( + SELECT json_group_array( + json_object('x',x,'y',random(),'z',hex(randomblob(50))) + ) + FROM c + ), + c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<5) +INSERT INTO t2(x) + SELECT json_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6) FROM array1, c2; +CREATE INDEX t2x1 ON t2(x->>'a'); +CREATE INDEX t2x2 ON t2(x->>'b'); +CREATE INDEX t2x3 ON t2(x->>'e'); +CREATE INDEX t2x4 ON t2(x->>'f'); +UPDATE t2 SET x=json_replace(x,'$.f',(x->>'f')+1); +UPDATE t2 SET x=json_set(x,'$.e',(x->>'f')-1); +UPDATE t2 SET x=json_remove(x,'$.d');