diff --git a/Makefile.in b/Makefile.in index 4339642eb6..ca23083fb8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -604,17 +604,35 @@ SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC FUZZERSHELL_OPT = FUZZCHECK_OPT += -I$(TOP)/test FUZZCHECK_OPT += -I$(TOP)/ext/recover -FUZZCHECK_OPT += -DSQLITE_OMIT_LOAD_EXTENSION -FUZZCHECK_OPT += -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 -FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 -FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4 -FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS3_PARENTHESIS -FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS5 -FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE -FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY -FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB -FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPT += \ + -DSQLITE_OSS_FUZZ \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_DESERIALIZE \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_FTS3_PARENTHESIS \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_GEOPOLY \ + -DSQLITE_ENABLE_MATH_FUNCTIONS \ + -DSQLITE_ENABLE_MEMSYS5 \ + -DSQLITE_ENABLE_NORMALIZE \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_SESSION \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + -DSQLITE_ENABLE_STAT4 \ + -DSQLITE_ENABLE_STMT_SCANSTATUS \ + -DSQLITE_MAX_MEMORY=50000000 \ + -DSQLITE_MAX_MMAP_SIZE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_PRINTF_PRECISION_LIMIT=1000 \ + -DSQLITE_PRIVATE="" + FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c @@ -681,6 +699,9 @@ fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) +fuzzcheck-asan$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) + $(LTLINK) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) + ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index 5528b8e722..a6b74cdc20 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1526,7 +1526,7 @@ TESTSRC = \ $(TOP)\ext\fts3\fts3_term.c \ $(TOP)\ext\fts3\fts3_test.c \ $(TOP)\ext\rbu\test_rbu.c \ - $(TOP)\ext\session\test_session.c + $(TOP)\ext\session\test_session.c # Statically linked extensions. # @@ -1671,6 +1671,33 @@ FUZZERSHELL_COMPILE_OPTS = FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DESERIALIZE +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS3_PARENTHESIS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MATH_FUNCTIONS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_NORMALIZE +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PREUPDATE_HOOK +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_SESSION +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMTVTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STAT4 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MMAP_SIZE=0 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE="" + FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION @@ -1789,6 +1816,9 @@ dbfuzz.exe: $(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H) fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +fuzzcheck-asan.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) /fsanitize=address $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + ossshell.exe: $(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) diff --git a/README.md b/README.md index 7ad42061c1..0df8b58c23 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ and most of the documentation are managed separately. ## Version Control -SQLite sources are managed using the +SQLite sources are managed using [Fossil](https://www.fossil-scm.org/), a distributed version control system that was specifically designed and written to support SQLite development. The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext. diff --git a/VERSION b/VERSION index 371986f6df..476cebe063 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.41.0 +3.42.0 diff --git a/autoconf/tea/configure.ac b/autoconf/tea/configure.ac index f6291d2557..e26780e2e9 100644 --- a/autoconf/tea/configure.ac +++ b/autoconf/tea/configure.ac @@ -19,7 +19,7 @@ dnl to configure the system for the local environment. # so that we create the export library with the dll. #----------------------------------------------------------------------- -AC_INIT([sqlite],[3.41.0]) +AC_INIT([sqlite],[3.42.0]) #-------------------------------------------------------------------- # Call TEA_INIT as the first TEA_ macro to set up initial vars. diff --git a/configure b/configure index 56e74f5ecc..29ca76b69e 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.41.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.42.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.41.0' -PACKAGE_STRING='sqlite 3.41.0' +PACKAGE_VERSION='3.42.0' +PACKAGE_STRING='sqlite 3.42.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1470,7 +1470,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.41.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.42.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1535,7 +1535,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.41.0:";; + short | recursive ) echo "Configuration of sqlite 3.42.0:";; esac cat <<\_ACEOF @@ -1665,7 +1665,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.41.0 +sqlite configure 3.42.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2084,7 +2084,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.41.0, which was +It was created by sqlite $as_me 3.42.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -12457,7 +12457,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.41.0, which was +This file was extended by sqlite $as_me 3.42.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12523,7 +12523,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.41.0 +sqlite config.status 3.42.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 6a727eaf5f..393f8a8717 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -2667,16 +2667,18 @@ static int fts3MsrBufferData( char *pList, i64 nList ){ - if( nList>pMsr->nBuffer ){ + if( (nList+FTS3_NODE_PADDING)>pMsr->nBuffer ){ char *pNew; - pMsr->nBuffer = nList*2; - pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, pMsr->nBuffer); + int nNew = nList*2 + FTS3_NODE_PADDING; + pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, nNew); if( !pNew ) return SQLITE_NOMEM; pMsr->aBuffer = pNew; + pMsr->nBuffer = nNew; } assert( nList>0 ); memcpy(pMsr->aBuffer, pList, nList); + memset(&pMsr->aBuffer[nList], 0, FTS3_NODE_PADDING); return SQLITE_OK; } diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index e7e7043c60..79a227cb44 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -199,6 +199,7 @@ struct Fts5Config { int ePattern; /* FTS_PATTERN_XXX constant */ /* Values loaded from the %_config table */ + int iVersion; /* fts5 file format 'version' */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ @@ -207,6 +208,7 @@ struct Fts5Config { int nHashSize; /* Bytes of memory for in-memory hash */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ + int bSecureDelete; /* 'secure-delete' */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; @@ -216,8 +218,11 @@ struct Fts5Config { #endif }; -/* Current expected value of %_config table 'version' field */ -#define FTS5_CURRENT_VERSION 4 +/* Current expected value of %_config table 'version' field. And +** the expected version if the 'secure-delete' option has ever been +** set on the table. */ +#define FTS5_CURRENT_VERSION 4 +#define FTS5_CURRENT_VERSION_SECUREDELETE 5 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 @@ -383,6 +388,7 @@ struct Fts5IndexIter { ** above. */ #define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 #define FTS5INDEX_QUERY_NOOUTPUT 0x0020 +#define FTS5INDEX_QUERY_SKIPHASH 0x0040 /* ** Create/destroy an Fts5Index object. diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 77f6d5baba..b178f47334 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -163,7 +163,7 @@ static int fts5HighlightCb( if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; iPos = p->iPos++; - if( p->iRangeEnd>0 ){ + if( p->iRangeEnd>=0 ){ if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; } @@ -175,7 +175,7 @@ static int fts5HighlightCb( } if( iPos==p->iter.iEnd ){ - if( p->iRangeEnd && p->iter.iStartiRangeStart ){ + if( p->iRangeEnd>=0 && p->iter.iStartiRangeStart ){ fts5HighlightAppend(&rc, p, p->zOpen, -1); } fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); @@ -186,7 +186,7 @@ static int fts5HighlightCb( } } - if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ + if( p->iRangeEnd>=0 && iPos==p->iRangeEnd ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; if( iPos>=p->iter.iStart && iPositer.iEnd ){ @@ -221,6 +221,7 @@ static void fts5HighlightFunction( memset(&ctx, 0, sizeof(HighlightContext)); ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); + ctx.iRangeEnd = -1; rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); if( ctx.zIn ){ @@ -406,6 +407,7 @@ static void fts5SnippetFunction( iCol = sqlite3_value_int(apVal[0]); ctx.zOpen = fts5ValueToText(apVal[1]); ctx.zClose = fts5ValueToText(apVal[2]); + ctx.iRangeEnd = -1; zEllips = fts5ValueToText(apVal[3]); nToken = sqlite3_value_int(apVal[4]); diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index ab1a846b12..df79605ca0 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -903,6 +903,18 @@ int sqlite3Fts5ConfigSetValue( rc = SQLITE_OK; *pbBadkey = 1; } + } + + else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){ + int bVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + bVal = sqlite3_value_int(pVal); + } + if( bVal<0 ){ + *pbBadkey = 1; + }else{ + pConfig->bSecureDelete = (bVal ? 1 : 0); + } }else{ *pbBadkey = 1; } @@ -947,15 +959,20 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ rc = sqlite3_finalize(p); } - if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ + if( rc==SQLITE_OK + && iVersion!=FTS5_CURRENT_VERSION + && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE + ){ rc = SQLITE_ERROR; if( pConfig->pzErrmsg ){ assert( 0==*pConfig->pzErrmsg ); - *pConfig->pzErrmsg = sqlite3_mprintf( - "invalid fts5 file format (found %d, expected %d) - run 'rebuild'", - iVersion, FTS5_CURRENT_VERSION + *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE ); } + }else{ + pConfig->iVersion = iVersion; } if( rc==SQLITE_OK ){ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index e4072db7aa..e87650d47d 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -407,7 +407,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ Fts5Parse sParse; memset(&sParse, 0, sizeof(sParse)); - if( *pp1 ){ + if( *pp1 && p2 ){ Fts5Expr *p1 = *pp1; int nPhrase = p1->nPhrase + p2->nPhrase; @@ -432,7 +432,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ } sqlite3_free(p2->apExprPhrase); sqlite3_free(p2); - }else{ + }else if( p2 ){ *pp1 = p2; } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 694cc16e45..cec150116b 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -302,6 +302,8 @@ struct Fts5Index { sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ + sqlite3_stmt *pDeleteFromIdx; + sqlite3_stmt *pDataVersion; i64 iStructVersion; /* data_version when pStruct read */ Fts5Structure *pStruct; /* Current db structure (or NULL) */ @@ -394,9 +396,6 @@ struct Fts5CResult { ** iLeafOffset: ** Byte offset within the current leaf that is the first byte of the ** position list data (one byte passed the position-list size field). -** rowid field of the current entry. Usually this is the size field of the -** position list data. The exception is if the rowid for the current entry -** is the last thing on the leaf page. ** ** pLeaf: ** Buffer containing current leaf page data. Set to NULL at EOF. @@ -985,6 +984,7 @@ static int fts5StructureDecode( */ static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ fts5StructureMakeWritable(pRc, ppStruct); + assert( (ppStruct!=0 && (*ppStruct)!=0) || (*pRc)!=SQLITE_OK ); if( *pRc==SQLITE_OK ){ Fts5Structure *pStruct = *ppStruct; int nLevel = pStruct->nLevel; @@ -1443,42 +1443,25 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ pLvl->bEof = 1; }else{ u8 *a = pLvl->pData->p; - i64 iVal; - int iLimit; - int ii; - int nZero = 0; - /* Currently iOff points to the first byte of a varint. This block - ** decrements iOff until it points to the first byte of the previous - ** varint. Taking care not to read any memory locations that occur - ** before the buffer in memory. */ - iLimit = (iOff>9 ? iOff-9 : 0); - for(iOff--; iOff>iLimit; iOff--){ - if( (a[iOff-1] & 0x80)==0 ) break; - } + pLvl->iOff = 0; + fts5DlidxLvlNext(pLvl); + while( 1 ){ + int nZero = 0; + int ii = pLvl->iOff; + u64 delta = 0; - fts5GetVarint(&a[iOff], (u64*)&iVal); - pLvl->iRowid -= iVal; - pLvl->iLeafPgno--; - - /* Skip backwards past any 0x00 varints. */ - for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){ - nZero++; - } - if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){ - /* The byte immediately before the last 0x00 byte has the 0x80 bit - ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80 - ** bytes before a[ii]. */ - int bZero = 0; /* True if last 0x00 counts */ - if( (ii-8)>=pLvl->iFirstOff ){ - int j; - for(j=1; j<=8 && (a[ii-j] & 0x80); j++); - bZero = (j>8); + while( a[ii]==0 ){ + nZero++; + ii++; } - if( bZero==0 ) nZero--; + ii += sqlite3Fts5GetVarint(&a[ii], &delta); + + if( ii>=iOff ) break; + pLvl->iLeafPgno += nZero+1; + pLvl->iRowid += delta; + pLvl->iOff = ii; } - pLvl->iLeafPgno -= nZero; - pLvl->iOff = iOff - nZero; } return pLvl->bEof; @@ -1674,7 +1657,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ i64 iOff = pIter->iLeafOffset; ASSERT_SZLEAF_OK(pIter->pLeaf); - if( iOff>=pIter->pLeaf->szLeaf ){ + while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; @@ -1773,10 +1756,12 @@ static void fts5SegIterInit( fts5SegIterSetNext(p, pIter); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; - fts5SegIterNextPage(p, pIter); + do { + fts5SegIterNextPage(p, pIter); + }while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 ); } - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && pIter->pLeaf ){ pIter->iLeafOffset = 4; assert( pIter->pLeaf!=0 ); assert_nc( pIter->pLeaf->nn>4 ); @@ -1970,7 +1955,7 @@ static void fts5SegIterNext_None( iOff = pIter->iLeafOffset; /* Next entry is on the next page */ - if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( p->rc || pIter->pLeaf==0 ) return; pIter->iRowid = 0; @@ -2163,7 +2148,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ Fts5Data *pLast = 0; int pgnoLast = 0; - if( pDlidx ){ + if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){ int iSegid = pIter->pSeg->iSegid; pgnoLast = fts5DlidxIterPgno(pDlidx); pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); @@ -2724,7 +2709,8 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ /* ** Move the seg-iter so that it points to the first rowid on page iLeafPgno. -** It is an error if leaf iLeafPgno does not exist or contains no rowids. +** It is an error if leaf iLeafPgno does not exist. Unless the db is +** a 'secure-delete' db, if it contains no rowids then this is also an error. */ static void fts5SegIterGotoPage( Fts5Index *p, /* FTS5 backend object */ @@ -2739,21 +2725,23 @@ static void fts5SegIterGotoPage( fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; pIter->iLeafPgno = iLeafPgno-1; - fts5SegIterNextPage(p, pIter); - assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); - if( p->rc==SQLITE_OK && ALWAYS(pIter->pLeaf!=0) ){ + while( p->rc==SQLITE_OK ){ int iOff; - u8 *a = pIter->pLeaf->p; - int n = pIter->pLeaf->szLeaf; - + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ) break; iOff = fts5LeafFirstRowidOff(pIter->pLeaf); - if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; - }else{ - iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; - fts5SegIterLoadNPos(p, pIter); + if( iOff>0 ){ + u8 *a = pIter->pLeaf->p; + int n = pIter->pLeaf->szLeaf; + if( iOff<4 || iOff>=n ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + fts5SegIterLoadNPos(p, pIter); + } + break; } } } @@ -3468,7 +3456,7 @@ static void fts5MultiIterNew( if( iLevel<0 ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; - nSeg += (p->pHash ? 1 : 0); + nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH)); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } @@ -3489,7 +3477,7 @@ static void fts5MultiIterNew( if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; - if( p->pHash ){ + if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){ /* Add a segment iterator for the current contents of the hash table. */ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); @@ -4244,7 +4232,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); - fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff,&pData->p[iOff]); + fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]); if( p->rc==SQLITE_OK ){ /* Set the szLeaf field */ fts5PutU16(&buf.p[2], (u16)buf.n); @@ -4522,16 +4510,16 @@ static void fts5IndexCrisismerge( ){ const int nCrisis = p->pConfig->nCrisisMerge; Fts5Structure *pStruct = *ppStruct; - int iLvl = 0; - - assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); - while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ - fts5IndexMergeLevel(p, &pStruct, iLvl, 0); - assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); - fts5StructurePromote(p, iLvl+1, pStruct); - iLvl++; + if( pStruct && pStruct->nLevel>0 ){ + int iLvl = 0; + while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); + fts5StructurePromote(p, iLvl+1, pStruct); + iLvl++; + } + *ppStruct = pStruct; } - *ppStruct = pStruct; } static int fts5IndexReturn(Fts5Index *p){ @@ -4565,6 +4553,413 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ return ret; } +/* +** Execute the SQL statement: +** +** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno); +** +** This is used when a secure-delete operation removes the last term +** from a segment leaf page. In that case the %_idx entry is removed +** too. This is done to ensure that if all instances of a token are +** removed from an fts5 database in secure-delete mode, no trace of +** the token itself remains in the database. +*/ +static void fts5SecureDeleteIdxEntry( + Fts5Index *p, /* FTS5 backend object */ + int iSegid, /* Id of segment to delete entry for */ + int iPgno /* Page number within segment */ +){ + if( iPgno!=1 ){ + assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE ); + if( p->pDeleteFromIdx==0 ){ + fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)", + p->pConfig->zDb, p->pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid); + sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno); + sqlite3_step(p->pDeleteFromIdx); + p->rc = sqlite3_reset(p->pDeleteFromIdx); + } + } +} + +/* +** This is called when a secure-delete operation removes a position-list +** that overflows onto segment page iPgno of segment pSeg. This function +** rewrites node iPgno, and possibly one or more of its right-hand peers, +** to remove this portion of the position list. +** +** Output variable (*pbLastInDoclist) is set to true if the position-list +** removed is followed by a new term or the end-of-segment, or false if +** it is followed by another rowid/position list. +*/ +static void fts5SecureDeleteOverflow( + Fts5Index *p, + Fts5StructureSegment *pSeg, + int iPgno, + int *pbLastInDoclist +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int pgno; + Fts5Data *pLeaf = 0; + assert( iPgno!=1 ); + + *pbLastInDoclist = 1; + for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){ + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); + int iNext = 0; + u8 *aPg = 0; + + pLeaf = fts5DataRead(p, iRowid); + if( pLeaf==0 ) break; + aPg = pLeaf->p; + + iNext = fts5GetU16(&aPg[0]); + if( iNext!=0 ){ + *pbLastInDoclist = 0; + } + if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){ + fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext); + } + + if( iNext==0 ){ + /* The page contains no terms or rowids. Replace it with an empty + ** page and move on to the right-hand peer. */ + const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; + assert_nc( bDetailNone==0 || pLeaf->nn==4 ); + if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty)); + fts5DataRelease(pLeaf); + pLeaf = 0; + }else if( bDetailNone ){ + break; + }else if( iNext>=pLeaf->szLeaf || iNext<4 ){ + p->rc = FTS5_CORRUPT; + break; + }else{ + int nShift = iNext - 4; + int nPg; + + int nIdx = 0; + u8 *aIdx = 0; + + /* Unless the current page footer is 0 bytes in size (in which case + ** the new page footer will be as well), allocate and populate a + ** buffer containing the new page footer. Set stack variables aIdx + ** and nIdx accordingly. */ + if( pLeaf->nn>pLeaf->szLeaf ){ + int iFirst = 0; + int i1 = pLeaf->szLeaf; + int i2 = 0; + + aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); + if( aIdx==0 ) break; + i1 += fts5GetVarint32(&aPg[i1], iFirst); + i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift); + if( i1nn ){ + memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1); + i2 += (pLeaf->nn-i1); + } + nIdx = i2; + } + + /* Modify the contents of buffer aPg[]. Set nPg to the new size + ** in bytes. The new page is always smaller than the old. */ + nPg = pLeaf->szLeaf - nShift; + memmove(&aPg[4], &aPg[4+nShift], nPg-4); + fts5PutU16(&aPg[2], nPg); + if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4); + if( nIdx>0 ){ + memcpy(&aPg[nPg], aIdx, nIdx); + nPg += nIdx; + } + sqlite3_free(aIdx); + + /* Write the new page to disk and exit the loop */ + assert( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, iRowid, aPg, nPg); + break; + } + } + fts5DataRelease(pLeaf); +} + +/* +** Completely remove the entry that pSeg currently points to from +** the database. +*/ +static void fts5DoSecureDelete( + Fts5Index *p, + Fts5SegIter *pSeg +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int iSegid = pSeg->pSeg->iSegid; + u8 *aPg = pSeg->pLeaf->p; + int nPg = pSeg->pLeaf->nn; + int iPgIdx = pSeg->pLeaf->szLeaf; + + u64 iDelta = 0; + u64 iNextDelta = 0; + int iNextOff = 0; + int iOff = 0; + int nIdx = 0; + u8 *aIdx = 0; + int bLastInDoclist = 0; + int iIdx = 0; + int iStart = 0; + int iKeyOff = 0; + int iPrevKeyOff = 0; + int iDelKeyOff = 0; /* Offset of deleted key, if any */ + + nIdx = nPg-iPgIdx; + aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + if( p->rc ) return; + memcpy(aIdx, &aPg[iPgIdx], nIdx); + + /* At this point segment iterator pSeg points to the entry + ** this function should remove from the b-tree segment. + ** + ** In detail=full or detail=column mode, pSeg->iLeafOffset is the + ** offset of the first byte in the position-list for the entry to + ** remove. Immediately before this comes two varints that will also + ** need to be removed: + ** + ** + the rowid or delta rowid value for the entry, and + ** + the size of the position list in bytes. + ** + ** Or, in detail=none mode, there is a single varint prior to + ** pSeg->iLeafOffset - the rowid or delta rowid value. + ** + ** This block sets the following variables: + ** + ** iStart: + ** iDelta: + */ + { + int iSOP; + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ + iStart = pSeg->iTermLeafOffset; + }else{ + iStart = fts5GetU16(&aPg[0]); + } + + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + assert_nc( iSOP<=pSeg->iLeafOffset ); + + if( bDetailNone ){ + while( iSOPiLeafOffset ){ + if( aPg[iSOP]==0x00 ) iSOP++; + if( aPg[iSOP]==0x00 ) iSOP++; + iStart = iSOP; + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + } + + iNextOff = iSOP; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + + }else{ + int nPos = 0; + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + while( iSOPiLeafOffset ){ + iStart = iSOP + (nPos/2); + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + } + assert_nc( iSOP==pSeg->iLeafOffset ); + iNextOff = pSeg->iLeafOffset + pSeg->nPos; + } + } + + iOff = iStart; + if( iNextOff>=iPgIdx ){ + int pgno = pSeg->iLeafPgno+1; + fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); + iNextOff = iPgIdx; + }else{ + /* Set bLastInDoclist to true if the entry being removed is the last + ** in its doclist. */ + for(iIdx=0, iKeyOff=0; iIdxiTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno + ){ + /* The entry being removed was the only position list in its + ** doclist. Therefore the term needs to be removed as well. */ + int iKey = 0; + for(iIdx=0, iKeyOff=0; iIdx(u32)iStart ) break; + iKeyOff += iVal; + } + + iDelKeyOff = iOff = iKeyOff; + if( iNextOff!=iPgIdx ){ + int nPrefix = 0; + int nSuffix = 0; + int nPrefix2 = 0; + int nSuffix2 = 0; + + iDelKeyOff = iNextOff; + iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); + iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + + if( iKey!=1 ){ + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + } + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + + nPrefix = MIN(nPrefix, nPrefix2); + nSuffix = (nPrefix2 + nSuffix2) - nPrefix; + + if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ + p->rc = FTS5_CORRUPT; + }else{ + if( iKey!=1 ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); + } + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); + if( nPrefix2>nPrefix ){ + memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); + iOff += (nPrefix2-nPrefix); + } + memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); + iOff += nSuffix2; + iNextOff += nSuffix2; + } + } + }else if( iStart==4 ){ + int iPgno; + + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; + + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; + + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } + } + fts5DataRelease(pTerm); + } + } + + if( p->rc==SQLITE_OK ){ + const int nMove = nPg - iNextOff; + int nShift = 0; + + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= (iNextOff - iOff); + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); + + nShift = iNextOff - iOff; + for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdxiOff ){ + iKeyOff -= nShift; + nShift = 0; + } + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); + iPrevKeyOff = iKeyOff; + } + } + + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + } + + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg); + } + sqlite3_free(aIdx); +} + +/* +** This is called as part of flushing a delete to disk in 'secure-delete' +** mode. It edits the segments within the database described by argument +** pStruct to remove the entries for term zTerm, rowid iRowid. +*/ +static void fts5FlushSecureDelete( + Fts5Index *p, + Fts5Structure *pStruct, + const char *zTerm, + i64 iRowid +){ + const int f = FTS5INDEX_QUERY_SKIPHASH; + int nTerm = (int)strlen(zTerm); + Fts5Iter *pIter = 0; /* Used to find term instance */ + + fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); + if( fts5MultiIterEof(p, pIter)==0 ){ + i64 iThis = fts5MultiIterRowid(pIter); + if( iThisrc==SQLITE_OK + && fts5MultiIterEof(p, pIter)==0 + && iRowid==fts5MultiIterRowid(pIter) + ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + fts5DoSecureDelete(p, pSeg); + } + } + + fts5MultiIterFree(pIter); +} + + /* ** Flush the contents of in-memory hash table iHash to a new level-0 ** segment on disk. Also update the corresponding structure record. @@ -4587,6 +4982,7 @@ static void fts5FlushOneHash(Fts5Index *p){ if( iSegid ){ const int pgsz = p->pConfig->pgsz; int eDetail = p->pConfig->eDetail; + int bSecureDelete = p->pConfig->bSecureDelete; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ @@ -4609,40 +5005,77 @@ static void fts5FlushOneHash(Fts5Index *p){ } while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ const char *zTerm; /* Buffer containing term */ + int nTerm; /* Size of zTerm in bytes */ const u8 *pDoclist; /* Pointer to doclist for this term */ int nDoclist; /* Size of doclist in bytes */ - /* Write the term for this entry to disk. */ + /* Get the term and doclist for this entry. */ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); - if( p->rc!=SQLITE_OK ) break; + nTerm = (int)strlen(zTerm); + if( bSecureDelete==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + assert( writer.bFirstRowidInPage==0 ); + } - assert( writer.bFirstRowidInPage==0 ); - if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ + int bTermWritten = !bSecureDelete; i64 iRowid = 0; - u64 iDelta = 0; + i64 iPrev = 0; int iOff = 0; /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ + iOff++; + continue; + } + } + } + + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + bTermWritten = 1; + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); + } if( writer.bFirstRowidInPage ){ fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); writer.bFirstRowidInPage = 0; fts5WriteDlidxAppend(p, &writer, iRowid); - if( p->rc!=SQLITE_OK ) break; }else{ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); } + if( p->rc!=SQLITE_OK ) break; assert( pBuf->n<=pBuf->nSpace ); + iPrev = iRowid; if( eDetail==FTS5_DETAIL_NONE ){ if( iOffnLevel==0 ){ - fts5StructureAddLevel(&p->rc, &pStruct); + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); + if( pgnoLast>0 ){ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); } - fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); - if( p->rc==SQLITE_OK ){ - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; - pSeg->iSegid = iSegid; - pSeg->pgnoFirst = 1; - pSeg->pgnoLast = pgnoLast; - pStruct->nSegment++; - } - fts5StructurePromote(p, 0, pStruct); } fts5IndexAutomerge(p, &pStruct, pgnoLast); @@ -5455,6 +5891,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); sqlite3_finalize(p->pDataVersion); + sqlite3_finalize(p->pDeleteFromIdx); sqlite3Fts5HashFree(p->pHash); sqlite3_free(p->zDataTbl); sqlite3_free(p); @@ -6085,6 +6522,7 @@ static void fts5IndexIntegrityCheckSegment( Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5Config *pConfig = p->pConfig; + int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE); sqlite3_stmt *pStmt = 0; int rc2; int iIdxPrevLeaf = pSeg->pgnoFirst-1; @@ -6120,7 +6558,19 @@ static void fts5IndexIntegrityCheckSegment( ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ if( pLeaf->nn<=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + + if( nIdxTerm==0 + && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE + && pLeaf->nn==pLeaf->szLeaf + && pLeaf->nn==4 + ){ + /* special case - the very first page in a segment keeps its %_idx + ** entry even if all the terms are removed from it by secure-delete + ** operations. */ + }else{ + p->rc = FTS5_CORRUPT; + } + }else{ int iOff; /* Offset of first term on leaf */ int iRowidOff; /* Offset of first rowid on leaf */ @@ -6184,9 +6634,12 @@ static void fts5IndexIntegrityCheckSegment( ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; - }else{ + }else if( bSecureDelete==0 || iRowidOff>0 ){ + i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); - if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; + if( iRowidrc = FTS5_CORRUPT; + } } fts5DataRelease(pLeaf); } diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 5392b3ba0f..13921ce49e 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1623,6 +1623,8 @@ static int fts5UpdateMethod( Fts5Config *pConfig = pTab->p.pConfig; int eType0; /* value_type() of apVal[0] */ int rc = SQLITE_OK; /* Return code */ + int bUpdateOrDelete = 0; + /* A transaction must be open when this is called. */ assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); @@ -1633,6 +1635,11 @@ static int fts5UpdateMethod( || sqlite3_value_type(apVal[0])==SQLITE_NULL ); assert( pTab->p.pConfig->pzErrmsg==0 ); + if( pConfig->pgsz==0 ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) return rc; + } + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; /* Put any active cursors into REQUIRE_SEEK state. */ @@ -1685,6 +1692,7 @@ static int fts5UpdateMethod( else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); + bUpdateOrDelete = 1; } /* INSERT or UPDATE */ @@ -1700,6 +1708,7 @@ static int fts5UpdateMethod( if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + bUpdateOrDelete = 1; } fts5StorageInsert(&rc, pTab, apVal, pRowid); } @@ -1728,10 +1737,24 @@ static int fts5UpdateMethod( rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); fts5StorageInsert(&rc, pTab, apVal, pRowid); } + bUpdateOrDelete = 1; } } } + if( rc==SQLITE_OK + && bUpdateOrDelete + && pConfig->bSecureDelete + && pConfig->iVersion==FTS5_CURRENT_VERSION + ){ + rc = sqlite3Fts5StorageConfigValue( + pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE + ); + if( rc==SQLITE_OK ){ + pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; + } + } + pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -2591,6 +2614,7 @@ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); + pTab->p.pConfig->pgsz = 0; return sqlite3Fts5StorageRollback(pTab->pStorage); } diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 0f371dcfd9..9c012932da 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -594,6 +594,10 @@ proc nearset_rc {aCol args} { list } +proc dump {tname} { + execsql_pp "SELECT * FROM ${tname}_idx" + execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data" +} #------------------------------------------------------------------------- # Code for a simple Tcl tokenizer that supports synonyms at query time. diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index a3ff330ef3..3d79295092 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -193,4 +193,34 @@ do_execsql_test 5.6 { } ;# foreach_detail_mode +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE t1 USING fts5(colA, colB); + INSERT INTO t1 VALUES('A B C', 'D E F'); +} + +do_execsql_test 6.1 { + SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'B'; +} {{A B C} {D E F} ...[B]...} +breakpoint +do_execsql_test 6.2 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'B'; +} {{A B C} {D E F} {D E...}} +do_execsql_test 6.3 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'B'; +} {{A B C} {D E F} {D...}} + +do_execsql_test 6.1 { + SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'A'; +} {{A B C} {D E F} [A]...} +breakpoint +do_execsql_test 6.2 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'A'; +} {{A B C} {D E F} {D E...}} +do_execsql_test 6.3 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'A'; +} {{A B C} {D E F} {D...}} + + + finish_test diff --git a/ext/fts5/test/fts5ak.test b/ext/fts5/test/fts5ak.test index 0a3cd6a783..e248f2a328 100644 --- a/ext/fts5/test/fts5ak.test +++ b/ext/fts5/test/fts5ak.test @@ -154,4 +154,30 @@ do_execsql_test 3.2 { } +# 2023-04-06 https://sqlite.org/forum/forumpost/cae4367d9b +# +# This is not a test of FTS5, but rather a test of the of what happens to +# prepared statements that encounter SQLITE_SCHEMA while other prepared +# statements are running. The original problem POC used FTS5, and so +# is seems reasonable to put the test here. +# +# The vdbeaux24.test module in TH3 also tests this same behavior but +# without requiring FTS5 or an other extension. +# +reset_db +db null NULL +do_execsql_test 4.0 { + CREATE TABLE t5(a PRIMARY KEY); + INSERT INTO t5 VALUES(0); + CREATE VIRTUAL TABLE t6 USING fts5(0); + DELETE FROM t6; + CREATE TABLE t7(x); + WITH cte(a) AS ( + SELECT a FROM t5 + WHERE ((0,0) IN (SELECT 0, LAG(0) OVER (PARTITION BY 0) FROM t6), 0) + < (a,0) + ) + SELECT max(a) FROM cte; +} NULL + finish_test diff --git a/ext/fts5/test/fts5corrupt7.test b/ext/fts5/test/fts5corrupt7.test new file mode 100644 index 0000000000..75995a7c03 --- /dev/null +++ b/ext/fts5/test/fts5corrupt7.test @@ -0,0 +1,99 @@ +# 2023 April 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt7 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} +sqlite3_fts5_may_be_corrupt 1 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} + +set doc [string repeat "a b " 30] + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO t1(rowid, x) VALUES(123, $doc); + INSERT INTO t1(rowid, x) VALUES(124, $doc); + COMMIT; +} + +execsql_pp { + SELECT id, fts5_decode(id, block), quote(block) FROM t1_data +} + +set rows [db eval { SELECT rowid FROM t1_data }] +db_save_and_close + +foreach r $rows { + db_restore_and_reopen + + proc edit_block {b} { + binary scan $b c* in + set out [lreplace $in 0 1 255 255] + binary format c* $out + } + db func edit_block edit_block + + do_execsql_test 1.2.$r.1 { + UPDATE t1_data SET block = edit_block(block) WHERE rowid=$r; + } + + do_execsql_test 1.2.$r.2 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } + + do_test 1.2.$r.3 { + catchsql { DELETE FROM t1 WHERE rowid=123; } + catchsql { DELETE FROM t1 WHERE rowid=124; } + set {} {} + } {} + + db close +} + +foreach r $rows { +set r 137438953475 + db_restore_and_reopen + + proc edit_block {b} { + binary scan $b c* in + set out [lreplace $in end end 127] + binary format c* $out + } + db func edit_block edit_block + + do_execsql_test 1.2.$r.1 { + UPDATE t1_data SET block = edit_block(block) WHERE rowid=$r; + } + + do_execsql_test 1.2.$r.2 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } + + do_test 1.2.$r.3 { + catchsql { DELETE FROM t1 WHERE rowid=124; } + catchsql { DELETE FROM t1 WHERE rowid=123; } + set {} {} + } {} + + db close +} + +finish_test diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index 416b4c8085..fb609e2c86 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -443,5 +443,33 @@ do_execsql_test -db db2 16.6 { SELECT * FROM x1 } {abc def} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 17.1 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="unicode61 separators 'X'"); +} +do_execsql_test 17.2 { + SELECT 0 FROM ft WHERE ft MATCH 'X' AND ft MATCH 'X' +} +do_execsql_test 17.3 { + SELECT 0 FROM ft('X') +} + +do_execsql_test 17.4 { + CREATE VIRTUAL TABLE t0 USING fts5(c0, t="trigram"); + INSERT INTO t0 VALUES('assertionfaultproblem'); +} +do_execsql_test 17.5 { + SELECT 0 FROM t0(0) WHERE c0 GLOB 0; +} {} + +do_execsql_test 17.5 { + SELECT c0 FROM t0 WHERE c0 GLOB '*f*'; +} {assertionfaultproblem} +do_execsql_test 17.5 { + SELECT c0 FROM t0 WHERE c0 GLOB '*faul*'; +} {assertionfaultproblem} + + finish_test diff --git a/ext/fts5/test/fts5secure.test b/ext/fts5/test/fts5secure.test new file mode 100644 index 0000000000..50d84cef79 --- /dev/null +++ b/ext/fts5/test/fts5secure.test @@ -0,0 +1,278 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure + +proc dump {tname} { + execsql_pp "SELECT * FROM ${tname}_idx" + execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data" +} + + +do_execsql_test 0.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + CREATE VIRTUAL TABLE v1 USING fts5vocab('t1', 'instance'); + INSERT INTO t1(rowid, ab) VALUES + (0,'abc'), (1,'abc'), (2,'abc'), (3,'abc'), (4,'def'); +} + +do_execsql_test 0.1 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 0.2 { + DELETE FROM t1 WHERE rowid=2; +} + +do_execsql_test 0.3 { + SELECT count(*) FROM t1_data +} 3 + +do_execsql_test 0.4 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 0.5 { + DELETE FROM t1 WHERE rowid=3; +} + +do_execsql_test 0.6 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 0.7 { + DELETE FROM t1 WHERE rowid=0; +} + +do_execsql_test 0.8 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#---------------------------------- + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5(ab); + INSERT INTO t2(rowid, ab) VALUES (5, 'key'), (6, 'value'); + INSERT INTO t2(t2, rank) VALUES('secure-delete', 1); +} + +#execsql_pp { SELECT id, quote(block) FROM t1_data } +#execsql_pp { SELECT segid, quote(term), pgno FROM t1_idx } + +do_execsql_test 1.1 { + DELETE FROM t2 WHERE rowid = 5; +} + +do_execsql_test 1.2 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +do_execsql_test 1.3 { + DELETE FROM t2 WHERE rowid = 6; +} + +do_execsql_test 1.4 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +do_execsql_test 1.5 { + SELECT * FROM t2('value'); + SELECT * FROM t2('v*'); +} + +do_execsql_test 1.6 { + SELECT * FROM t2('value') ORDER BY rowid DESC; + SELECT * FROM t2('v*') ORDER BY rowid DESC; +} +execsql_pp { + SELECT id, quote(block) FROM t2_data; +} + +#---------------------------------- + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(ab); + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', 'instance'); + INSERT INTO ft(rowid, ab) VALUES + (1, 'one'), + (2, 'two'), + (3, 'three'), + (4, 'four'), + (5, 'one one'), + (6, 'one two'), + (7, 'one three'), + (8, 'one four'), + (9, 'two one'), + (10, 'two two'), + (11, 'two three'), + (12, 'two four'), + (13, 'three one'), + (14, 'three two'), + (15, 'three three'), + (16, 'three four'); +} + +do_execsql_test 2.1 { + SELECT count(*) FROM ft_data; +} {3} + +do_execsql_test 2.2 { + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 2.3 { + DELETE FROM ft WHERE rowid=9; +} + +do_execsql_test 2.4 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 2.5 { + DELETE FROM ft WHERE ab LIKE '%two%' +} + +do_execsql_test 2.6 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 2.7 { + SELECT count(*) FROM ft_data; +} {3} + +#---------------------------------- +reset_db + +set ::vocab { + one two three four five six seven eight nine ten + eleven twelve thirteen fourteen fifteen sixteen + seventeen eighteen nineteen twenty +} +proc rnddoc {} { + set nVocab [llength $::vocab] + set ret [list] + for {set ii 0} {$ii < 8} {incr ii} { + lappend ret [lindex $::vocab [expr int(abs(rand()) * $nVocab)]] + } + set ret +} + +proc contains {list val} { + expr {[lsearch $list $val]>=0} +} + +foreach {tn pgsz} { + 2 64 + 1 1000 +} { + reset_db + db function rnddoc rnddoc + db function contains contains + + expr srand(1) + + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); + WITH s(i) AS ( + VALUES(1) UNION SELECT i+1 FROM s WHERE i<20 + ) + INSERT INTO t1 SELECT rnddoc() FROM s; + } + + do_execsql_test 3.$tn.1 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } + + foreach {rowid} { + 6 16 3 4 9 14 13 7 20 15 19 10 11 2 5 18 17 1 12 8 + } { + + do_execsql_test 3.$tn.2.$rowid { + DELETE FROM t1 WHERE rowid=$rowid; + } + do_execsql_test 3.$tn.2.$rowid.ic { + INSERT INTO t1(t1) VALUES('integrity-check'); + } + + foreach v $::vocab { + do_execsql_test 3.$tn.2.$rowid.q.$v { + SELECT rowid FROM t1($v) + } [db eval {SELECT rowid FROM t1 WHERE contains(x, $v)}] + + do_execsql_test 3.$tn.2.$rowid.q.$v.DESC { + SELECT rowid FROM t1($v) ORDER BY 1 DESC + } [db eval {SELECT rowid FROM t1 WHERE contains(x, $v) ORDER BY 1 DESC}] + } + } +} + +do_execsql_test 3.3 { + INSERT INTO t1(x) VALUES('optimize'); + INSERT INTO t1(t1) VALUES('optimize'); + SELECT count(*) FROM t1_data; +} {3} + +#---------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +set L1 [string repeat abcdefghij 10] +set L2 [string repeat 1234567890 10] + +do_execsql_test 4.1 { + INSERT INTO t1 VALUES('aa' || $L1 || ' ' || $L2); +} +do_execsql_test 4.2 { + DELETE FROM t1 WHERE rowid=1 +} +do_execsql_test 4.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#---------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +set doc "aa [string repeat {abc } 60]" + +do_execsql_test 5.1 { + BEGIN; + INSERT INTO t1 VALUES($doc); + INSERT INTO t1 VALUES('aa abc'); + COMMIT; +} + +do_execsql_test 5.2 { + DELETE FROM t1 WHERE rowid = 1; +} + +do_execsql_test 5.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2 +do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2 + + +finish_test + diff --git a/ext/fts5/test/fts5secure2.test b/ext/fts5/test/fts5secure2.test new file mode 100644 index 0000000000..04ff66219c --- /dev/null +++ b/ext/fts5/test/fts5secure2.test @@ -0,0 +1,87 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure2 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(col); + INSERT INTO ft VALUES('data for the table'); + INSERT INTO ft VALUES('more of the same'); + INSERT INTO ft VALUES('and extra data'); +} + +do_execsql_test 1.1 { + SELECT * FROM ft_config +} {version 4} + +do_execsql_test 1.2 { + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); + SELECT * FROM ft_config; +} {secure-delete 1 version 4} + +do_execsql_test 1.3 { + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); + SELECT * FROM ft_config; +} {secure-delete 1 version 4} + +do_execsql_test 1.4 { + DELETE FROM ft WHERE rowid=2; + SELECT * FROM ft_config; +} {secure-delete 1 version 5} + +do_execsql_test 1.5 { + SELECT rowid, col FROM ft('data'); +} {1 {data for the table} 3 {and extra data}} + +db close +sqlite3 db test.db + +do_execsql_test 1.6 { + SELECT rowid, col FROM ft('data'); +} {1 {data for the table} 3 {and extra data}} + +#------------------------------------------------------------------------ + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(col); + INSERT INTO ft VALUES('one zero one one zero'); + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 2.1 { + SELECT count(*) FROM ft_data WHERE block=X'00000004'; +} {0} + +do_execsql_test 2.2 { + UPDATE ft SET col = 'zero one zero zero one' WHERE rowid=1; +} + +do_execsql_test 2.3 { + SELECT count(*) FROM ft_data WHERE block=X'00000004'; +} {1} + +do_execsql_test 2.4 { + INSERT INTO ft VALUES('one zero zero one'); + DELETE FROM ft WHERE rowid=1; +} + +do_execsql_test 2.5 { + SELECT count(*) FROM ft_data WHERE block=X'00000004'; +} {2} + + +finish_test + + diff --git a/ext/fts5/test/fts5secure3.test b/ext/fts5/test/fts5secure3.test new file mode 100644 index 0000000000..bc56e08200 --- /dev/null +++ b/ext/fts5/test/fts5secure3.test @@ -0,0 +1,166 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# TESTRUNNER: slow +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure3 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(col); + INSERT INTO ft VALUES('data for the table'); + INSERT INTO ft VALUES('more of the same'); + INSERT INTO ft VALUES('and extra data'); + + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO ft(rowid, col) VALUES(0, 'the next data'); + DELETE FROM ft WHERE rowid=1; + DELETE FROM ft WHERE rowid=2; + INSERT INTO ft(rowid, col) VALUES(6, 'with some more of the same data'); + COMMIT; +} + +do_execsql_test 1.2 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + BEGIN; + INSERT INTO t1 VALUES('the start'); +} +do_test 2.1 { + for {set i 0} {$i < 1000} {incr i} { + execsql { INSERT INTO t1 VALUES('the ' || hex(randomblob(3))) } + } + execsql { + INSERT INTO t1 VALUES('the end'); + COMMIT; + } +} {} + +do_execsql_test 2.2 { + DELETE FROM t1 WHERE rowid BETWEEN 2 AND 1000; +} + +do_execsql_test 2.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 2.6 { + INSERT INTO t1(rowid, x) VALUES(500, 'middle'); + INSERT INTO t1(rowid, x) VALUES(501, 'value'); + SELECT * FROM t1('the middle'); +} + +do_execsql_test 2.7 { + INSERT INTO t1(t1) VALUES('optimize'); +} + +do_execsql_test 2.8 { + SELECT count(*) FROM t1_data +} 4 + +#execsql_pp { SELECT id, quote(block), fts5_decode(id, block) FROM t1_data; } + +#------------------------------------------------------------------------- +# Tests with large/small rowid values. +# + +reset_db + +expr srand(0) + +set vocab { + Popper Poppins Popsicle Porfirio Porrima Porsche + Porter Portia Portland Portsmouth Portugal Portuguese + Poseidon Post PostgreSQL Potemkin Potomac Potsdam + Pottawatomie Potter Potts Pound Poussin Powell + PowerPC PowerPoint Powers Powhatan Poznan Prada + Prado Praetorian Prague Praia Prakrit Pratchett + Pratt Pravda Praxiteles Preakness Precambrian Preminger + Premyslid Prensa Prentice Pres Presbyterian Presbyterianism +} +proc newdoc {} { + for {set i 0} {$i<8} {incr i} { + lappend ret [lindex $::vocab [expr int(abs(rand()) * [llength $::vocab])]] + } + set ret +} +db func newdoc newdoc + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE fff USING fts5(y); + INSERT INTO fff(fff, rank) VALUES('pgsz', 64); + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + INSERT INTO fff(fff, rank) VALUES('secure-delete', 1); +} + +proc lshuffle {in} { + set out [list] + while {[llength $in]>0} { + set idx [expr int(abs(rand()) * [llength $in])] + lappend out [lindex $in $idx] + set in [lreplace $in $idx $idx] + } + set out +} + +#dump fff + +set iTest 1 +foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] { + #if {$iTest==1} { dump fff } + #if {$iTest==1} { breakpoint } + do_execsql_test 3.1.$iTest.$ii { + DELETE FROM fff WHERE rowid=$ii; + } + #if {$iTest==1} { dump fff } + if {($iTest % 20)==0} { + do_execsql_test 3.1.$iTest.$ii.ic { + INSERT INTO fff(fff) VALUES('integrity-check'); + } + } + #if {$iTest==1} { break } + incr iTest +} + +#execsql_pp { SELECT rowid FROM fff('post') ORDER BY rowid ASC } +#breakpoint +#execsql_pp { +# SELECT rowid FROM fff('post') ORDER BY rowid DESC +#} +# +#dump fff + + +finish_test + diff --git a/ext/fts5/test/fts5secure4.test b/ext/fts5/test/fts5secure4.test new file mode 100644 index 0000000000..7588a34683 --- /dev/null +++ b/ext/fts5/test/fts5secure4.test @@ -0,0 +1,170 @@ +# 2023 April 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +return_if_no_fts5 +set ::testprefix fts5secure4 + +#------------------------------------------------------------------------- +# Test using the 'delete' command to attempt to delete a token that +# is not present in the index in secure-delete mode. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, content=x1); + + CREATE TABLE x1(rowid INTEGER PRIMARY KEY, a, b); + INSERT INTO x1 VALUES + (1, 'hello world', 'today xyz'), + (2, 'not the day', 'crunch crumble and chomp'), + (3, 'one', 'two'); + INSERT INTO t1(t1) VALUES('rebuild'); +} + +do_execsql_test 1.1 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 1.2 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 4, 'nosuchtoken', ''); +} + +do_execsql_test 1.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 1.4 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 1, 'crunch', ''); +} + +do_execsql_test 1.5 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 3, 'crunch', ''); +} + +do_execsql_test 1.6 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 1.7 { +CREATE VIRTUAL TABLE y1 USING fts5(xx, prefix='1,2'); +INSERT INTO y1(y1, rank) VALUES('pgsz', 64); +INSERT INTO y1(y1, rank) VALUES('secure-delete', 1); +} +do_execsql_test 1.8 { + BEGIN; + INSERT INTO y1(rowid, xx) VALUES(1, 'abc def'); + INSERT INTO y1(rowid, xx) VALUES(2, 'reallyreallylongtoken'); + COMMIT; +} +do_execsql_test 1.9 { + DELETE FROM y1 WHERE rowid=1; + INSERT INTO y1(y1) VALUES('integrity-check'); +} + +do_execsql_test 1.10 { + CREATE VIRTUAL TABLE w1 USING fts5(ww, content=""); + INSERT INTO w1(rowid, ww) VALUES(123, ''); +} +do_catchsql_test 1.11 { + INSERT INTO w1(w1, rowid, ww) VALUES('delete', 123, 'xyz'); +} {1 {database disk image is malformed}} +do_catchsql_test 1.12 { + DROP TABLE w1; + CREATE VIRTUAL TABLE w1 USING fts5(ww, content=""); + INSERT INTO w1(rowid, ww) VALUES(123, ''); + DELETE FROM w1_data WHERE id>10; + INSERT INTO w1(w1, rowid, ww) VALUES('delete', 123, 'xyz'); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# Test using secure-delete with detail=none or detail=col. +# +foreach {tn d} {1 full 2 none 3 column} { + reset_db + do_execsql_test 2.$tn.1 " + CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='10,20'); + INSERT INTO x1(x1, rank) VALUES('pgsz', 64); + INSERT INTO x1(x1, rank) VALUES('secure-delete', 1); + " + + do_execsql_test 2.$tn.2 { + BEGIN; + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + COMMIT; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + + do_execsql_test 2.$tn.3 { + DELETE FROM x1 WHERE rowid IN (2, 4, 6); + INSERT INTO x1(x1) VALUES('integrity-check'); + } + + do_execsql_test 2.$tn.4 { + DELETE FROM x1 WHERE rowid IN (1, 3, 5); + INSERT INTO x1(x1) VALUES('integrity-check'); + } + + do_execsql_test 2.$tn.5 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO x1 + SELECT 'seems to be', 'used brew to', 'everything is working' FROM s + UNION ALL + SELECT 'used brew to', 'everything is working', 'seems to be' FROM s + UNION ALL + SELECT 'everything is working', 'seems to be', 'used brew to' FROM s + UNION ALL + SELECT 'abc', 'zzz', 'a b c d' + UNION ALL + SELECT 'z', 'z', 'z' FROM s + } + + do_test 2.$tn.6 { + for {set i 300} {$i > 200} {incr i -1} { + execsql { + DELETE FROM x1 WHERE rowid=$i; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + } + } {} + + do_test 2.$tn.7 { + for {set i 1} {$i < 100} {incr i} { + execsql { + DELETE FROM x1 WHERE rowid=$i; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + } + } {} + + do_test 2.$tn.8 { + foreach i [db eval {SELECT rowid FROM x1}] { + execsql { + DELETE FROM x1 WHERE rowid=$i; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + } + } {} + + do_execsql_test 2.$tn.9 { + SELECT * FROM x1 + } {} +} + + + +finish_test + diff --git a/ext/fts5/test/fts5secure5.test b/ext/fts5/test/fts5secure5.test new file mode 100644 index 0000000000..ca8570211c --- /dev/null +++ b/ext/fts5/test/fts5secure5.test @@ -0,0 +1,129 @@ +# 2023 April 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +return_if_no_fts5 +set ::testprefix fts5secure5 +return_if_no_fts5 + +proc dump {} { + execsql_pp { + SELECT id, quote(block), fts5_decode_none(id, block) FROM ft1_data + } +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO ft1(rowid, a) VALUES(1, 'abcd'); + INSERT INTO ft1(rowid, a) VALUES(2, 'abcd'); + INSERT INTO ft1(rowid, a) VALUES(3, 'abcd'); + COMMIT; +} +do_execsql_test 1.2 { + DELETE FROM ft1 WHERE rowid=1; +} +do_execsql_test 1.3 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 1.4 { + DELETE FROM ft1 WHERE rowid=3; +} +do_execsql_test 1.5 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 1.6 { + DELETE FROM ft1 WHERE rowid=3; +} +do_execsql_test 1.7 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 2.1 { + BEGIN; + INSERT INTO ft1(rowid, a) VALUES(1, 'abcd one'); + INSERT INTO ft1(rowid, a) VALUES(2, 'abcd two'); + INSERT INTO ft1(rowid, a) VALUES(3, 'abcd two'); + INSERT INTO ft1(rowid, a) VALUES(4, 'abcd two'); + INSERT INTO ft1(rowid, a) VALUES(5, 'abcd three'); + COMMIT; +} + +do_execsql_test 2.2a { + DELETE FROM ft1 WHERE rowid=3; +} +do_execsql_test 2.2b { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 2.3a { + DELETE FROM ft1 WHERE rowid=2; +} +do_execsql_test 2.3b { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 2.4a { + DELETE FROM ft1 WHERE rowid=4; +} +do_execsql_test 2.4b { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none, prefix=1); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); + INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); +} +do_execsql_test 3.1 { + BEGIN; + INSERT INTO ft1(a) VALUES('c'); + COMMIT; +} +do_execsql_test 3.2 { + DELETE FROM ft1 WHERE rowid IN (1); + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); + INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); + + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500 + ) + INSERT INTO ft1 SELECT 'abcdefg' FROM s; +} + +do_test 4.1 { + for {set i 500} {$i > 0} {incr i -1} { + execsql { DELETE FROM ft1 WHERE rowid=$i } + execsql { INSERT INTO ft1(ft1) VALUES('integrity-check') } + } +} {} + +finish_test + diff --git a/ext/fts5/test/fts5secure6.test b/ext/fts5/test/fts5secure6.test new file mode 100644 index 0000000000..e561a43f7d --- /dev/null +++ b/ext/fts5/test/fts5secure6.test @@ -0,0 +1,55 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure6 + +db progress 1 progress_handler +set ::PHC 0 +proc progress_handler {args} { + incr ::PHC + if {($::PHC % 100000)==0} breakpoint + return 0 +} + +proc setup {} { + db eval { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING fts5(x); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t1 SELECT 'a b c d e f g h i j k' FROM s; + } +} + +foreach {tn sd} { + 1 0 + 2 1 +} { + setup + do_execsql_test 1.$tn.0 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) + } + set PHC 0 + do_execsql_test 1.$tn.1 { DELETE FROM t1; } + set phc($tn) $PHC +} + +do_test 1.3 { + expr $phc(1)*5 < $phc(2) +} {1} + + +finish_test + diff --git a/ext/fts5/test/fts5securefault.test b/ext/fts5/test/fts5securefault.test new file mode 100644 index 0000000000..63874ece5d --- /dev/null +++ b/ext/fts5/test/fts5securefault.test @@ -0,0 +1,225 @@ +# 2023 April 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5securefault + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +return_if_no_fts5 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(rowid, ab) VALUES + (0, 'abc'), (1, 'abc'), (2, 'abc'), (3, 'abc'), (4, 'def'); +} +faultsim_save_and_close + +do_faultsim_test 1.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid=2 } +} -test { + faultsim_test_result {0 {}} +} +do_faultsim_test 1.2 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid IN(0, 1, 2, 3, 4) } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set big [string repeat abcdefghij 5] +set big2 [string repeat klmnopqrst 5] +set doc "$big $big2" + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<4 + ) + INSERT INTO t1(rowid, ab) SELECT i, $doc FROM s; +} +faultsim_save_and_close + +do_faultsim_test 2.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 3 } + execsql { DELETE FROM t1 WHERE rowid = 4 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set big [string repeat abcdefghij 5] +set big2 [string repeat klmnopqrst 5] +set doc "$big $big2" + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<25 + ) + INSERT INTO t1(rowid, ab) SELECT i, $doc FROM s; + + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + DELETE FROM t1 WHERE rowid BETWEEN 3 AND 23; +} +faultsim_save_and_close + +do_faultsim_test 3.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 24 } + execsql { DELETE FROM t1 WHERE rowid = 25 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set doc [string repeat "tok " 400] + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + INSERT INTO t1(rowid, ab) VALUES(1, $doc), (2, $doc), (3, $doc); +} +faultsim_save_and_close + +do_faultsim_test 4.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 2 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db + +set doc1 [string repeat "abc " 10] +set doc2 [string repeat "def " 10] + +do_test 5.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + BEGIN; + } + for {set i 0} {$i < 50} {incr i} { + execsql { + INSERT INTO t1(rowid, ab) VALUES($i, 'abcdefg'); + } + } + execsql { + INSERT INTO t1(rowid, ab) VALUES(105, 'def'); + COMMIT; + } +} {} +faultsim_save_and_close + +do_faultsim_test 5.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 105 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +do_test 6.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + BEGIN; + INSERT INTO t1(rowid, ab) VALUES(1, 'abcdefg'); + INSERT INTO t1(rowid, ab) VALUES(2, 'abcdefg'); + INSERT INTO t1(rowid, ab) VALUES(3, 'abcdefg'); + COMMIT; + } +} {} +faultsim_save_and_close + +do_faultsim_test 6.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { + UPDATE t1 SET ab='abcdefg' WHERE rowid=2; + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +do_test 7.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} {} +faultsim_save_and_close + +do_faultsim_test 7.1 -faults oom* -prep { + faultsim_restore_and_reopen + set big1 "[string repeat x 50] [string repeat y 50] [string repeat z 50]" + execsql { + BEGIN; + INSERT INTO t1 VALUES($big1); + } +} -body { + execsql { COMMIT } +} -test { + faultsim_test_result {0 {}} +} + + +finish_test diff --git a/ext/fts5/test/fts5version.test b/ext/fts5/test/fts5version.test index 60ec81c03d..79fd94e6bc 100644 --- a/ext/fts5/test/fts5version.test +++ b/ext/fts5/test/fts5version.test @@ -38,20 +38,20 @@ do_execsql_test 1.3 { sqlite3_db_config db DEFENSIVE 0 do_execsql_test 1.4 { - UPDATE t1_config set v=5 WHERE k='version'; + UPDATE t1_config set v=6 WHERE k='version'; } do_test 1.5 { db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } -} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 6, expected 4 or 5) - run 'rebuild'}} do_test 1.6 { db close sqlite3 db test.db catchsql { INSERT INTO t1 VALUES('x y z') } -} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 6, expected 4 or 5) - run 'rebuild'}} do_test 1.7 { sqlite3_db_config db DEFENSIVE 0 @@ -59,7 +59,75 @@ do_test 1.7 { db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } -} {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}} + +do_test 1.8 { + sqlite3_db_config db DEFENSIVE 0 + execsql { INSERT INTO t1_config VALUES('version', 4) } + execsql { INSERT INTO t1(t1, rank) VALUES('secure-delete', 1) } +} {} + +do_execsql_test 1.10 { + SELECT * FROM t1_config +} {secure-delete 1 version 4} + +do_execsql_test 1.11 { + INSERT INTO t1(rowid, one) VALUES(123, 'one two three'); + DELETE FROM t1 WHERE rowid=123; + SELECT * FROM t1_config +} {secure-delete 1 version 5} + +do_execsql_test 1.11 { + INSERT INTO t1(t1) VALUES('rebuild'); + SELECT * FROM t1_config +} {secure-delete 1 version 4} + +do_execsql_test 1.12 { + SELECT * FROM t1_config +} {secure-delete 1 version 4} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xyz USING fts5(x); + INSERT INTO xyz(rowid, x) VALUES + (1, 'one document'), + (2, 'two document'), + (3, 'three document'), + (4, 'four document'), + (5, 'five document'), + (6, 'six document'); + + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1); + SELECT v FROM xyz_config WHERE k='version'; +} {4} + +do_execsql_test 2.1 { + BEGIN; + INSERT INTO xyz(rowid, x) VALUES(7, 'seven document'); + SAVEPOINT one; + DELETE FROM xyz WHERE rowid = 4; +} + +do_execsql_test 2.2 { + SELECT v FROM xyz_config WHERE k='version'; +} {5} + +do_execsql_test 2.3 { + ROLLBACK TO one; + SELECT v FROM xyz_config WHERE k='version'; +} {4} + + +do_execsql_test 2.4 { + DELETE FROM xyz WHERE rowid = 3; + COMMIT; + SELECT v FROM xyz_config WHERE k='version'; +} {5} + + finish_test + diff --git a/ext/misc/base64.c b/ext/misc/base64.c old mode 100755 new mode 100644 index 69eff61811..bc7a976abc --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -76,6 +76,7 @@ typedef unsigned char u8; #define U8_TYPEDEF #endif +/* Decoding table, ASCII (7-bit) value to base 64 digit value or other */ static const u8 b64DigitValues[128] = { /* HT LF VT FF CR */ ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, @@ -147,9 +148,9 @@ static char* toBase64( u8 *pIn, int nbIn, char *pOut ){ } /* Skip over text which is not base64 numeral(s). */ -static char * skipNonB64( char *s ){ +static char * skipNonB64( char *s, int nc ){ char c; - while( (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; + while( nc-- > 0 && (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; return s; } @@ -158,7 +159,7 @@ static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 && *pIn!=PAD_CHAR ){ static signed char nboi[] = { 0, 0, 1, 2, 3 }; - char *pUse = skipNonB64(pIn); + char *pUse = skipNonB64(pIn, ncIn); unsigned long qv = 0L; int nti, nbo, nac; ncIn -= (pUse - pIn); @@ -218,9 +219,16 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "blob expanded to base64 too big", -1); return; } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } cBuf = sqlite3_malloc(nc); if( !cBuf ) goto memFail; - bBuf = (u8*)sqlite3_value_blob(av[0]); nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); break; @@ -233,9 +241,16 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ }else if( nb<1 ){ nb = 1; } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } bBuf = sqlite3_malloc(nb); if( !bBuf ) goto memFail; - cBuf = (char *)sqlite3_value_text(av[0]); nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); break; diff --git a/ext/misc/base85.c b/ext/misc/base85.c index 5ec136dbc6..e7ef0a04c9 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -140,9 +140,9 @@ static u8 base85DigitValue( char c ){ #define B85_DARK_MAX 80 -static char * skipNonB85( char *s ){ +static char * skipNonB85( char *s, int nc ){ char c; - while( (c = *s) && !IS_B85(c) ) ++s; + while( nc-- > 0 && (c = *s) && !IS_B85(c) ) ++s; return s; } @@ -212,7 +212,7 @@ static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 ){ static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; - char *pUse = skipNonB85(pIn); + char *pUse = skipNonB85(pIn, ncIn); unsigned long qv = 0L; int nti, nbo; ncIn -= (pUse - pIn); @@ -297,9 +297,16 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "blob expanded to base85 too big", -1); return; } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } cBuf = sqlite3_malloc(nc); if( !cBuf ) goto memFail; - bBuf = (u8*)sqlite3_value_blob(av[0]); nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); break; @@ -312,9 +319,16 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ }else if( nb<1 ){ nb = 1; } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } bBuf = sqlite3_malloc(nb); if( !bBuf ) goto memFail; - cBuf = (char *)sqlite3_value_text(av[0]); nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); break; diff --git a/ext/misc/randomjson.c b/ext/misc/randomjson.c new file mode 100644 index 0000000000..3a6f545fe6 --- /dev/null +++ b/ext/misc/randomjson.c @@ -0,0 +1,202 @@ +/* +** 2023-04-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements a the random_json(SEED) and +** random_json5(SEED) functions. Given a numeric SEED value, these +** routines generate pseudo-random JSON or JSON5, respectively. The +** same value is always generated for the same seed. +** +** These SQL functions are intended for testing. They do not have any +** practical real-world use, that we know of. +** +** COMPILE: +** +** gcc --shared -fPIC -o randomjson.so -I. ext/misc/randomjson.c +** +** USING FROM THE CLI: +** +** .load ./randomjson +** SELECT random_json(1); +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/* Pseudo-random number generator */ +typedef struct Prng { + unsigned int x, y; +} Prng; + +/* Reseed the PRNG */ +static void prngSeed(Prng *p, unsigned int iSeed){ + p->x = iSeed | 1; + p->y = iSeed; +} + +/* Extract a random number */ +static unsigned int prngInt(Prng *p){ + p->x = (p->x>>1) ^ ((1+~(p->x&1)) & 0xd0000001); + p->y = p->y*1103515245 + 12345; + return p->x ^ p->y; +} + +static const char *azJsonAtoms[] = { + /* JSON /* JSON-5 */ + "0", "0", + "1", "1", + "-1", "-1", + "2", "+2", + "3", "3", + "2.5", "2.5", + "0.75", ".75", + "-4.0e2", "-4.e2", + "5.0e-3", "+5e-3", + "0", "0x0", + "512", "0x200", + "256", "+0x100", + "-2748", "-0xabc", + "true", "true", + "false", "false", + "null", "null", + "9.0e999", "Infinity", + "-9.0e999", "-Infinity", + "9.0e999", "+Infinity", + "null", "NaN", + "-0.0005123", "-0.0005123", + "4.35e-3", "+4.35e-3", + "\"gem\\\"hay\"", "\"gem\\\"hay\"", + "\"icy'joy\"", "'icy\\'joy\'", + "\"keylog\"", "\"key\\\nlog\"", + "\"mix\\\\\\tnet\"", "\"mix\\\\\\tnet\"", + "{}", "{}", + "[]", "[]", + "[]", "[/*empty*/]", + "{}", "{//empty\n}", + "\"ask\"", "\"ask\"", + "\"bag\"", "\"bag\"", + "\"can\"", "\"can\"", + "\"day\"", "\"day\"", + "\"end\"", "'end'", + "\"fly\"", "\"fly\"", + "\"\"", "\"\"", +}; +static const char *azJsonTemplate[] = { + /* JSON JSON-5 */ + "{\"a\":%,\"b\":%,\"c\":%}", "{a:%,b:%,c:%}", + "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"e\":%}", "{a:%,b:%,c:%,d:%,e:%}", + "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}", "{a:%,b:%,c:%,d:%,\"\":%}", + "{\"d\":%}", "{d:%}", + "{\"eeee\":%, \"ffff\":%}", "{eeee:% /*and*/, ffff:%}", + "{\"$g\":%,\"_h_\":%}", "{$g:%,_h_:%,}", + "{\"x\":%,\n \"y\":%}", "{\"x\":%,\n \"y\":%}", + "{\"a b c d\":%,\"e\":%,\"f\":%,\"x\":%,\"y\":%}", + "{\"a b c d\":%,e:%,f:%,x:%,y:%}", + "{\"Z\":%}", "{Z:%,}", + "[%]", "[%,]", + "[%,%]", "[%,%]", + "[%,%,%]", "[%,%,%,]", + "[%,%,%,%]", "[%,%,%,%]", + "[%,%,%,%,%]", "[%,%,%,%,%]", +}; + +#define count(X) (sizeof(X)/sizeof(X[0])) + +#define STRSZ 10000 + +static void jsonExpand( + const char *zSrc, + char *zDest, + Prng *p, + int eType, /* 0 for JSON, 1 for JSON5 */ + unsigned int r /* Growth probability 0..1000. 0 means no growth */ +){ + unsigned int i, j, k; + const char *z; + size_t n; + + j = 0; + if( zSrc==0 ){ + k = prngInt(p)%(count(azJsonTemplate)/2); + k = k*2 + eType; + zSrc = azJsonTemplate[k]; + } + if( strlen(zSrc)>=STRSZ/10 ) r = 0; + for(i=0; zSrc[i]; i++){ + if( zSrc[i]!='%' ){ + if( j= 0 ) +** for each produced value (independent of production time ordering.) +** +** All parameters must be either integer or convertable to integer. +** The start parameter is required. +** The stop parameter defaults to (1<<32)-1 (aka 4294967295 or 0xffffffff) +** The step parameter defaults to 1 and 0 is treated as 1. +** ** Examples: ** ** SELECT * FROM generate_series(0,100,5); @@ -28,6 +40,14 @@ ** ** Integers 20 through 29. ** +** SELECT * FROM generate_series(0,-100,-5); +** +** Integers 0 -5 -10 ... -100. +** +** SELECT * FROM generate_series(0,-1); +** +** Empty sequence. +** ** HOW IT WORKS ** ** The generate_series "function" is really a virtual table with the @@ -40,6 +60,9 @@ ** step HIDDEN ** ); ** +** The virtual table also has a rowid, logically equivalent to n+1 where +** "n" is the ascending integer in the aforesaid production definition. +** ** Function arguments in queries against this virtual table are translated ** into equality constraints against successive hidden columns. In other ** words, the following pairs of queries are equivalent to each other: @@ -72,9 +95,96 @@ SQLITE_EXTENSION_INIT1 #include #include +#include #ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return that member of a generate_series(...) sequence whose 0-based +** index is ix. The 0th member is given by smBase. The sequence members +** progress per ix increment by smStep. +*/ +static sqlite3_int64 genSeqMember(sqlite3_int64 smBase, + sqlite3_int64 smStep, + sqlite3_uint64 ix){ + if( ix>=(sqlite3_uint64)LLONG_MAX ){ + /* Get ix into signed i64 range. */ + ix -= (sqlite3_uint64)LLONG_MAX; + smBase += LLONG_MAX * smStep; + } + return smBase + ((sqlite3_int64)ix)*smStep; +} +typedef unsigned char u8; + +typedef struct SequenceSpec { + sqlite3_int64 iBase; /* Starting value ("start") */ + sqlite3_int64 iTerm; /* Given terminal value ("stop") */ + sqlite3_int64 iStep; /* Increment ("step") */ + sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ + sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ + sqlite3_int64 iValueNow; /* Current value during generation */ + u8 isNotEOF; /* Sequence generation not exhausted */ + u8 isReversing; /* Sequence is being reverse generated */ +} SequenceSpec; + +/* +** Prepare a SequenceSpec for use in generating an integer series +** given initialized iBase, iTerm and iStep values. Sequence is +** initialized per given isReversing. Other members are computed. +*/ +void setupSequence( SequenceSpec *pss ){ + pss->uSeqIndexMax = 0; + pss->isNotEOF = 0; + if( pss->iTerm < pss->iBase ){ + sqlite3_uint64 nuspan = (sqlite3_uint64)(pss->iBase-pss->iTerm); + if( pss->iStep<0 ){ + pss->isNotEOF = 1; + if( nuspan==ULONG_MAX ){ + pss->uSeqIndexMax = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1; + }else if( pss->iStep>LLONG_MIN ){ + pss->uSeqIndexMax = nuspan/-pss->iStep; + } + } + }else if( pss->iTerm > pss->iBase ){ + sqlite3_uint64 puspan = (sqlite3_uint64)(pss->iTerm-pss->iBase); + if( pss->iStep>0 ){ + pss->isNotEOF = 1; + pss->uSeqIndexMax = puspan/pss->iStep; + } + }else if( pss->iTerm == pss->iBase ){ + pss->isNotEOF = 1; + pss->uSeqIndexMax = 0; + } + pss->uSeqIndexNow = (pss->isReversing)? pss->uSeqIndexMax : 0; + pss->iValueNow = (pss->isReversing) + ? genSeqMember(pss->iBase, pss->iStep, pss->uSeqIndexMax) + : pss->iBase; +} + +/* +** Progress sequence generator to yield next value, if any. +** Leave its state to either yield next value or be at EOF. +** Return whether there is a next value, or 0 at EOF. +*/ +int progressSequence( SequenceSpec *pss ){ + if( !pss->isNotEOF ) return 0; + if( pss->isReversing ){ + if( pss->uSeqIndexNow > 0 ){ + pss->uSeqIndexNow--; + pss->iValueNow -= pss->iStep; + }else{ + pss->isNotEOF = 0; + } + }else{ + if( pss->uSeqIndexNow < pss->uSeqIndexMax ){ + pss->uSeqIndexNow++; + pss->iValueNow += pss->iStep; + }else{ + pss->isNotEOF = 0; + } + } + return pss->isNotEOF; +} /* series_cursor is a subclass of sqlite3_vtab_cursor which will ** serve as the underlying representation of a cursor that scans @@ -83,12 +193,7 @@ SQLITE_EXTENSION_INIT1 typedef struct series_cursor series_cursor; struct series_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - int isDesc; /* True to count down rather than up */ - sqlite3_int64 iRowid; /* The rowid */ - sqlite3_int64 iValue; /* Current value ("value") */ - sqlite3_int64 mnValue; /* Mimimum value ("start") */ - sqlite3_int64 mxValue; /* Maximum value ("stop") */ - sqlite3_int64 iStep; /* Increment ("step") */ + SequenceSpec ss; /* (this) Derived class data */ }; /* @@ -170,12 +275,7 @@ static int seriesClose(sqlite3_vtab_cursor *cur){ */ static int seriesNext(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - if( pCur->isDesc ){ - pCur->iValue -= pCur->iStep; - }else{ - pCur->iValue += pCur->iStep; - } - pCur->iRowid++; + progressSequence( & pCur->ss ); return SQLITE_OK; } @@ -191,10 +291,10 @@ static int seriesColumn( series_cursor *pCur = (series_cursor*)cur; sqlite3_int64 x = 0; switch( i ){ - case SERIES_COLUMN_START: x = pCur->mnValue; break; - case SERIES_COLUMN_STOP: x = pCur->mxValue; break; - case SERIES_COLUMN_STEP: x = pCur->iStep; break; - default: x = pCur->iValue; break; + case SERIES_COLUMN_START: x = pCur->ss.iBase; break; + case SERIES_COLUMN_STOP: x = pCur->ss.iTerm; break; + case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; + default: x = pCur->ss.iValueNow; break; } sqlite3_result_int64(ctx, x); return SQLITE_OK; @@ -207,7 +307,7 @@ static int seriesColumn( */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; - *pRowid = pCur->iRowid; + *pRowid = ((sqlite3_int64)pCur->ss.uSeqIndexNow + 1); return SQLITE_OK; } @@ -217,14 +317,10 @@ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int seriesEof(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - if( pCur->isDesc ){ - return pCur->iValue < pCur->mnValue; - }else{ - return pCur->iValue > pCur->mxValue; - } + return !pCur->ss.isNotEOF; } -/* True to cause run-time checking of the start=, stop=, and/or step= +/* True to cause run-time checking of the start=, stop=, and/or step= ** parameters. The only reason to do this is for testing the ** constraint checking logic for virtual tables in the SQLite core. */ @@ -235,7 +331,7 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ /* ** This method is called to "rewind" the series_cursor object back ** to the first row of output. This method is always called at least -** once prior to any call to seriesColumn() or seriesRowid() or +** once prior to any call to seriesColumn() or seriesRowid() or ** seriesEof(). ** ** The query plan selected by seriesBestIndex is passed in the idxNum @@ -255,7 +351,7 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ ** (so that seriesEof() will return true) if the table is empty. */ static int seriesFilter( - sqlite3_vtab_cursor *pVtabCursor, + sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStrUnused, int argc, sqlite3_value **argv ){ @@ -263,46 +359,41 @@ static int seriesFilter( int i = 0; (void)idxStrUnused; if( idxNum & 1 ){ - pCur->mnValue = sqlite3_value_int64(argv[i++]); + pCur->ss.iBase = sqlite3_value_int64(argv[i++]); }else{ - pCur->mnValue = 0; + pCur->ss.iBase = 0; } if( idxNum & 2 ){ - pCur->mxValue = sqlite3_value_int64(argv[i++]); + pCur->ss.iTerm = sqlite3_value_int64(argv[i++]); }else{ - pCur->mxValue = 0xffffffff; + pCur->ss.iTerm = 0xffffffff; } if( idxNum & 4 ){ - pCur->iStep = sqlite3_value_int64(argv[i++]); - if( pCur->iStep==0 ){ - pCur->iStep = 1; - }else if( pCur->iStep<0 ){ - pCur->iStep = -pCur->iStep; + pCur->ss.iStep = sqlite3_value_int64(argv[i++]); + if( pCur->ss.iStep==0 ){ + pCur->ss.iStep = 1; + }else if( pCur->ss.iStep<0 ){ if( (idxNum & 16)==0 ) idxNum |= 8; } }else{ - pCur->iStep = 1; + pCur->ss.iStep = 1; } for(i=0; imnValue = 1; - pCur->mxValue = 0; + pCur->ss.iBase = 1; + pCur->ss.iTerm = 0; + pCur->ss.iStep = 1; break; } } if( idxNum & 8 ){ - pCur->isDesc = 1; - pCur->iValue = pCur->mxValue; - if( pCur->iStep>0 ){ - pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep; - } + pCur->ss.isReversing = pCur->ss.iStep > 0; }else{ - pCur->isDesc = 0; - pCur->iValue = pCur->mnValue; + pCur->ss.isReversing = pCur->ss.iStep < 0; } - pCur->iRowid = 1; + setupSequence( &pCur->ss ); return SQLITE_OK; } diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c index 765c691811..ba3ea581f8 100644 --- a/ext/misc/shathree.c +++ b/ext/misc/shathree.c @@ -10,7 +10,8 @@ ** ****************************************************************************** ** -** This SQLite extension implements functions that compute SHA3 hashes. +** This SQLite extension implements functions that compute SHA3 hashes +** in the way described by the (U.S.) NIST FIPS 202 SHA-3 Standard. ** Two SQL functions are implemented: ** ** sha3(X,SIZE) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 480fbe3990..9b49fb4df6 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -1097,7 +1097,10 @@ static int zipfileColumn( ** it to be a directory either if the mode suggests so, or if ** the final character in the name is '/'. */ u32 mode = pCDS->iExternalAttr >> 16; - if( !(mode & S_IFDIR) && pCDS->zFile[pCDS->nFile-1]!='/' ){ + if( !(mode & S_IFDIR) + && pCDS->nFile>=1 + && pCDS->zFile[pCDS->nFile-1]!='/' + ){ sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC); } } diff --git a/ext/rbu/rbuexlock.test b/ext/rbu/rbuexlock.test index 27fd6c4ba2..28ab308fc0 100644 --- a/ext/rbu/rbuexlock.test +++ b/ext/rbu/rbuexlock.test @@ -207,5 +207,86 @@ do_test 3.4.0 { } SQLITE_OK rbu close +#------------------------------------------------------------------------- +reset_db +forcedelete rbu1.db +forcedelete rbu2.db + +sqlite3 rbu rbu1.db +do_execsql_test -db rbu 4.1 { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 'one', 0); +} +rbu close +sqlite3 rbu rbu2.db +do_execsql_test -db rbu 4.2 { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES(2, 'two', 0); +} +rbu close + +do_execsql_test 4.3 { + CREATE TABLE t1(a PRIMARY KEY, b); +} +db close + +do_test 4.4 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu1.db + rbu step + rbu state +} {oal} + +sqlite3 cons test.db +do_execsql_test -db cons 4.5 { + SELECT * FROM t1 +} {} + +do_test 4.6 { rbu step ; rbu state } {oal} +do_test 4.7 { rbu step ; rbu state } {move} +do_execsql_test -db cons 4.8 { + SELECT * FROM t1 +} {} +do_test 4.9 { rbu step ; rbu state } {checkpoint} +do_test 4.10 { + catchsql { SELECT * FROM t1 } cons +} {1 {database is locked}} +do_test 4.11 { rbu step ; rbu state } {checkpoint} +do_test 4.11 { rbu step ; rbu state } {done} +rbu close + +do_test 4.12 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one}} + +do_test 4.13 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu2.db + rbu step + rbu state +} {oal} + +do_test 4.14 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one}} + +do_test 4.15 { rbu step ; rbu state } {oal} +do_test 4.16 { rbu step ; rbu state } {move} + +do_test 4.17 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one}} + +do_test 4.18 { rbu step ; rbu state } {checkpoint} +do_test 4.19 { + catchsql { SELECT * FROM t1 } cons +} {1 {database is locked}} +do_test 4.20 { rbu step ; rbu state } {checkpoint} +do_test 4.21 { rbu step ; rbu state } {done} +rbu close + +do_test 4.22 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one 2 two}} + +cons close finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 2db66f67ac..f65ef83864 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3141,6 +3141,11 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff); } +/* +** This value is copied from the definition of ZIPVFS_CTRL_FILE_POINTER +** in zipvfs.h. +*/ +#define RBU_ZIPVFS_CTRL_FILE_POINTER 230439 /* ** Take an EXCLUSIVE lock on the database file. Return SQLITE_OK if @@ -3149,7 +3154,11 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ static int rbuLockDatabase(sqlite3 *db){ int rc = SQLITE_OK; sqlite3_file *fd = 0; - sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + + sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd); + if( fd==0 ){ + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + } if( fd->pMethods ){ rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); diff --git a/ext/recover/dbdata.c b/ext/recover/dbdata.c index da02b754b2..878a61f1d8 100644 --- a/ext/recover/dbdata.c +++ b/ext/recover/dbdata.c @@ -167,6 +167,7 @@ static int dbdataConnect( (void)argc; (void)argv; (void)pzErr; + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); if( rc==SQLITE_OK ){ pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); if( pTab==0 ){ @@ -512,10 +513,14 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); if( rc!=SQLITE_OK ) return rc; - if( pCsr->aPage ) break; + if( pCsr->aPage && pCsr->nPage>=256 ) break; + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; } + + assert( iOff+3+2<=pCsr->nPage ); pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); } @@ -750,8 +755,7 @@ static int dbdataGetEncoding(DbdataCursor *pCsr){ int nPg1 = 0; u8 *aPg1 = 0; rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1); - assert( rc!=SQLITE_OK || nPg1==0 || nPg1>=512 ); - if( rc==SQLITE_OK && nPg1>0 ){ + if( rc==SQLITE_OK && nPg1>=(56+4) ){ pCsr->enc = get_uint32(&aPg1[56]); } sqlite3_free(aPg1); @@ -809,8 +813,6 @@ static int dbdataFilter( } if( rc==SQLITE_OK ){ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); - }else{ - pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } /* Try to determine the encoding of the db by inspecting the header @@ -819,6 +821,10 @@ static int dbdataFilter( rc = dbdataGetEncoding(pCsr); } + if( rc!=SQLITE_OK ){ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + } + if( rc==SQLITE_OK ){ rc = dbdataNext(pCursor); } diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index fef1bf90f4..26894eaca2 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -316,5 +316,18 @@ do_execsql_test 16.9 { COMMIT; } {1 2 3 4} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 17.1 { + CREATE TABLE t(a, PRIMARY KEY(a, a COLLATE NOCASE)) WITHOUT ROWID; + INSERT INTO t VALUES('abc'); + INSERT INTO t VALUES('def'); +} +do_test 17.2 { + set R [sqlite3_recover_init db main test.db2] + $R run + list [catch { $R finish } msg] $msg +} {0 {}} + finish_test diff --git a/ext/recover/recoverbuild.test b/ext/recover/recoverbuild.test new file mode 100644 index 0000000000..d119be7fde --- /dev/null +++ b/ext/recover/recoverbuild.test @@ -0,0 +1,42 @@ +# 2023 February 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recoverbuild + + +# The following tests verify that if the recovery extension is used with +# a build that does not support the sqlite_dbpage table, the error message +# is "no such table: sqlite_dbpage", and not something more generic. +# +reset_db +create_null_module db sqlite_dbpage +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1 VALUES(123, 'one hundred and twenty three'); +} + +forcedelete test.db2 +do_test 1.1 { + set R [sqlite3_recover_init db main test.db2] +} {/sqlite_recover.*/} + +do_test 1.2 { + $R run +} {1} + +do_test 1.3 { + list [catch { $R finish } msg] $msg +} {1 {no such table: sqlite_dbpage}} + +finish_test + diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 8306e8ed8e..29fff0e7e2 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -1106,7 +1106,7 @@ static void recoverAddTable( int iField = sqlite3_column_int(pStmt, 0); int iCol = sqlite3_column_int(pStmt, 1); - assert( iFieldnCol && iColnCol ); + assert( iColnCol ); pNew->aCol[iCol].iField = iField; pNew->bIntkey = 0; diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 38d09d2497..da5e2a97a8 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -471,16 +471,17 @@ struct RtreeMatchArg { ** at run-time. */ #ifndef SQLITE_BYTEORDER -#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) -# define SQLITE_BYTEORDER 1234 -#elif defined(sparc) || defined(__ppc__) -# define SQLITE_BYTEORDER 4321 -#else -# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ -#endif +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) || \ + defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 +# else +# define SQLITE_BYTEORDER 0 +# endif #endif diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index b6dfe992a6..1cbb2c6e87 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -104,6 +104,7 @@ do_eqp_test rtree6.2.4.1 { } { QUERY PLAN |--SCAN t1 VIRTUAL TABLE INDEX 2:C0E1 + |--BLOOM FILTER ON t2 (v=?) `--SEARCH t2 USING AUTOMATIC COVERING INDEX (v=?) } do_eqp_test rtree6.2.4.2 { @@ -111,6 +112,7 @@ do_eqp_test rtree6.2.4.2 { } { QUERY PLAN |--SCAN t1 VIRTUAL TABLE INDEX 2:C0E1 + |--BLOOM FILTER ON t2 (v=?) `--SEARCH t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?) } diff --git a/ext/session/changebatch1.test b/ext/session/changebatch1.test index 2fbe368947..16053d26d2 100644 --- a/ext/session/changebatch1.test +++ b/ext/session/changebatch1.test @@ -195,7 +195,7 @@ do_test 3.1.1 { sqlite3changebatch cb db cb add $c1 cb add $c2 -} {SQLITE_CONSTRAINT} +} {SQLITE_OK} do_test 3.1.2 { cb delete } {} diff --git a/ext/session/session2.test b/ext/session/session2.test index 806687745e..207b98740e 100644 --- a/ext/session/session2.test +++ b/ext/session/session2.test @@ -191,7 +191,7 @@ do_common_sql { } foreach {tn sql} [string map {%T1% t1 %T2% t2 %T3% t3 %T4% t4} $set_of_tests] { - do_then_apply_sql $sql + do_then_apply_sql -ignorenoop $sql do_test 2.$tn { compare_db db db2 } {} } @@ -598,7 +598,7 @@ do_common_sql { INSERT INTO t1 SELECT NULL, 0, 0, 0, 0, 0 FROM s } -do_then_apply_sql { +do_then_apply_sql -ignorenoop { UPDATE t1 SET f=f+1 WHERE a=1; UPDATE t1 SET e=e+1 WHERE a=2; UPDATE t1 SET e=e+1, f=f+1 WHERE a=3; diff --git a/ext/session/sessionD.test b/ext/session/sessionD.test index ec652e2d6b..9fccbfa96f 100644 --- a/ext/session/sessionD.test +++ b/ext/session/sessionD.test @@ -41,30 +41,6 @@ proc scksum {db dbname} { return [md5 $txt] } -proc do_diff_test {tn setup} { - reset_db - forcedelete test.db2 - execsql { ATTACH 'test.db2' AS aux } - execsql $setup - - sqlite3session S db main - foreach tbl [db eval {SELECT name FROM sqlite_master WHERE type='table'}] { - S attach $tbl - S diff aux $tbl - } - - set C [S changeset] - S delete - - sqlite3 db2 test.db2 - sqlite3changeset_apply db2 $C "" - uplevel do_test $tn.1 [list {execsql { PRAGMA integrity_check } db2}] ok - db2 close - - set cksum [scksum db main] - uplevel do_test $tn.2 [list {scksum db aux}] [list $cksum] -} - # Ensure that the diff produced by comparing the current contents of [db] # with itself is empty. proc do_empty_diff_test {tn} { diff --git a/ext/session/sessionG.test b/ext/session/sessionG.test index 58ea17d2ee..1ebcc926a5 100644 --- a/ext/session/sessionG.test +++ b/ext/session/sessionG.test @@ -34,7 +34,7 @@ do_test 1.0 { INSERT INTO t1 VALUES(2, 'two'); INSERT INTO t1 VALUES(3, 'three'); } - do_then_apply_sql { + do_then_apply_sql -ignorenoop { DELETE FROM t1 WHERE a=1; INSERT INTO t1 VALUES(4, 'one'); } @@ -42,7 +42,7 @@ do_test 1.0 { } {} do_test 1.1 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { DELETE FROM t1 WHERE a=4; INSERT INTO t1 VALUES(1, 'one'); } @@ -51,7 +51,7 @@ do_test 1.1 { do_test 1.2 { execsql { INSERT INTO t1 VALUES(5, 'five') } db2 - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t1 VALUES(11, 'eleven'); INSERT INTO t1 VALUES(12, 'five'); } @@ -82,7 +82,7 @@ do_test 2.2.1 { # It is not possible to apply the changeset generated by the following # SQL, as none of the three updated rows may be updated as part of the # first pass. - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t1 SET b=0 WHERE a=1; UPDATE t1 SET b=1 WHERE a=2; UPDATE t1 SET b=2 WHERE a=3; @@ -109,7 +109,7 @@ do_test 3.1 { } {} do_test 3.3 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t1 SET b=4 WHERE a=3; UPDATE t1 SET b=3 WHERE a=2; UPDATE t1 SET b=2 WHERE a=1; @@ -118,7 +118,7 @@ do_test 3.3 { } {} do_test 3.4 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t1 SET b=1 WHERE a=1; UPDATE t1 SET b=2 WHERE a=2; UPDATE t1 SET b=3 WHERE a=3; @@ -148,7 +148,7 @@ do_test 4.1 { } {} do_test 4.2 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t1 SET b=4 WHERE a=3; UPDATE t1 SET b=3 WHERE a=2; UPDATE t1 SET b=2 WHERE a=1; @@ -161,7 +161,7 @@ do_test 4.2 { } {} do_test 4.3 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t1 SET b=1 WHERE a=1; UPDATE t1 SET b=2 WHERE a=2; UPDATE t1 SET b=3 WHERE a=3; @@ -191,7 +191,7 @@ do_execsql_test -db db2 5.0.2 { } do_test 5.1 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t2 VALUES(4, 5, 6); INSERT INTO t3 VALUES(7, 8, 9); diff --git a/ext/session/sessionH.test b/ext/session/sessionH.test index 36851e9337..c8b41600ea 100644 --- a/ext/session/sessionH.test +++ b/ext/session/sessionH.test @@ -25,7 +25,7 @@ do_test 1.0 { do_common_sql { CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)); } - do_then_apply_sql { + do_then_apply_sql -ignorenoop { WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERe i<10000 ) diff --git a/ext/session/session_common.tcl b/ext/session/session_common.tcl index c52ac457c0..3ff84f1c5e 100644 --- a/ext/session/session_common.tcl +++ b/ext/session/session_common.tcl @@ -52,6 +52,7 @@ proc do_conflict_test {tn args} { proc bgerror {args} { set ::background_error $args } sqlite3session S db main + S object_config rowid 1 foreach t $O(-tables) { S attach $t } execsql $O(-sql) @@ -81,6 +82,7 @@ proc changeset_from_sql {sql {dbname main}} { } set rc [catch { sqlite3session S db $dbname + S object_config rowid 1 db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { S attach $name } @@ -112,24 +114,59 @@ proc patchset_from_sql {sql {dbname main}} { return $patchset } -proc do_then_apply_sql {sql {dbname main}} { - proc xConflict args { return "OMIT" } +# Usage: do_then_apply_sql ?-ignorenoop? SQL ?DBNAME? +# +proc do_then_apply_sql {args} { + + set bIgnoreNoop 0 + set a1 [lindex $args 0] + if {[string length $a1]>1 && [string first $a1 -ignorenoop]==0} { + set bIgnoreNoop 1 + set args [lrange $args 1 end] + } + + if {[llength $args]!=1 && [llength $args]!=2} { + error "usage: do_then_apply_sql ?-ignorenoop? SQL ?DBNAME?" + } + + set sql [lindex $args 0] + if {[llength $args]==1} { + set dbname main + } else { + set dbname [lindex $args 1] + } + + set ::n_conflict 0 + proc xConflict args { incr ::n_conflict ; return "OMIT" } set rc [catch { sqlite3session S db $dbname + S object_config rowid 1 db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { S attach $name } db eval $sql - sqlite3changeset_apply db2 [S changeset] xConflict + set ::changeset [S changeset] + sqlite3changeset_apply db2 $::changeset xConflict } msg] catch { S delete } - if {$rc} {error $msg} + + if {$bIgnoreNoop} { + set nSave $::n_conflict + set ::n_conflict 0 + proc xConflict args { incr ::n_conflict ; return "OMIT" } + sqlite3changeset_apply_v2 -ignorenoop db2 $::changeset xConflict + if {$::n_conflict!=$nSave} { + error "-ignorenoop problem ($::n_conflict $nSave)..." + } + } } proc do_iterator_test {tn tbl_list sql res} { sqlite3session S db main + S object_config rowid 1 + if {[llength $tbl_list]==0} { S attach * } foreach t $tbl_list {S attach $t} @@ -139,6 +176,7 @@ proc do_iterator_test {tn tbl_list sql res} { foreach v $res { lappend r $v } set x [list] +# set ::c [S changeset] ; execsql_pp { SELECT quote($::c) } sqlite3session_foreach c [S changeset] { lappend x $c } uplevel do_test $tn [list [list set {} $x]] [list $r] @@ -213,3 +251,49 @@ proc number_name {n} { if {$txt==""} {set txt zero} return $txt } + +proc scksum {db dbname} { + + if {$dbname=="temp"} { + set master sqlite_temp_master + } else { + set master $dbname.sqlite_master + } + + set alltab [$db eval "SELECT name FROM $master WHERE type='table'"] + set txt [$db eval "SELECT * FROM $master ORDER BY type,name,sql"] + foreach tab $alltab { + set cols [list] + db eval "PRAGMA $dbname.table_info = $tab" x { + lappend cols "quote($x(name))" + } + set cols [join $cols ,] + append txt [db eval "SELECT $cols FROM $dbname.$tab ORDER BY $cols"] + } + return [md5 $txt] +} + +proc do_diff_test {tn setup} { + reset_db + forcedelete test.db2 + execsql { ATTACH 'test.db2' AS aux } + execsql $setup + + sqlite3session S db main + S object_config rowid 1 + foreach tbl [db eval {SELECT name FROM sqlite_master WHERE type='table'}] { + S attach $tbl + S diff aux $tbl + } + + set C [S changeset] + S delete + + sqlite3 db2 test.db2 + sqlite3changeset_apply db2 $C "" + uplevel do_test $tn.1 [list {execsql { PRAGMA integrity_check } db2}] ok + db2 close + + set cksum [scksum db main] + uplevel do_test $tn.2 [list {scksum db aux}] [list $cksum] +} diff --git a/ext/session/sessionat.test b/ext/session/sessionat.test index e3f9e31ed7..e14901e8b5 100644 --- a/ext/session/sessionat.test +++ b/ext/session/sessionat.test @@ -110,7 +110,7 @@ eval [string map [list %WR% $trailing] { CREATE TABLE t3(a, b, c DEFAULT 'D', PRIMARY KEY(b)) %WR%; } do_test $tn.3.2 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t3 VALUES(1, 2); INSERT INTO t3 VALUES(3, 4); INSERT INTO t3 VALUES(5, 6); @@ -118,7 +118,7 @@ eval [string map [list %WR% $trailing] { db2 eval {SELECT * FROM t3} } {1 2 D 3 4 D 5 6 D} do_test $tn.3.3 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t3 SET a=45 WHERE b=4; DELETE FROM t3 WHERE a=5; }; @@ -253,7 +253,7 @@ eval [string map [list %WR% $trailing] { CREATE TABLE t8(a PRIMARY KEY, b, c, d DEFAULT 'D', e DEFAULT 'E'); } - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t8 VALUES(1, 2, 3); INSERT INTO t8 VALUES(4, 5, 6); } @@ -264,7 +264,7 @@ eval [string map [list %WR% $trailing] { SELECT * FROM t8 } {1 2 3 D E 4 5 6 D E} - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t8 SET c=45 WHERE a=4; } do_execsql_test $tn.7.3.1 { @@ -282,7 +282,7 @@ eval [string map [list %WR% $trailing] { do_execsql_test -db db2 $tn.8.1 { CREATE TABLE t9(a PRIMARY KEY, b, c, d, e, f, g, h, i, j, k, l); } - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t9 VALUES(1, 2, 3, 4, 5, 6, 7, 8); } do_then_apply_sql { @@ -291,7 +291,7 @@ eval [string map [list %WR% $trailing] { do_execsql_test -db db2 $tn.8.2 { SELECT * FROM t9 } {1 2 3 4 5 6 7 450 {} {} {} {}} - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t9 SET h=NULL } do_execsql_test -db db2 $tn.8.2 { diff --git a/ext/session/sessionbig.test b/ext/session/sessionbig.test index 80ce00a0f7..462e21f61f 100644 --- a/ext/session/sessionbig.test +++ b/ext/session/sessionbig.test @@ -43,7 +43,7 @@ do_execsql_test -db db2 1.1 { } do_test 1.2 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); @@ -71,7 +71,7 @@ do_test 1.3 { do_test 1.4 { set rc [catch { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); diff --git a/ext/session/sessionfault.test b/ext/session/sessionfault.test index be6c4568ce..96e966b41d 100644 --- a/ext/session/sessionfault.test +++ b/ext/session/sessionfault.test @@ -44,7 +44,7 @@ do_faultsim_test 1.1 -faults oom-* -prep { faultsim_restore_and_reopen sqlite3 db2 test.db2 } -body { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t1 VALUES('a string value', 8, 9); UPDATE t1 SET c = 10 WHERE a = 1; DELETE FROM t1 WHERE a = 4; diff --git a/ext/session/sessionfault2.test b/ext/session/sessionfault2.test index dd00eaa1c8..a2dc39e437 100644 --- a/ext/session/sessionfault2.test +++ b/ext/session/sessionfault2.test @@ -132,7 +132,7 @@ do_faultsim_test 1.1 -faults oom-* -prep { faultsim_restore_and_reopen sqlite3 db2 test.db2 } -body { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO sqlite_stat1 VALUES('x', 'y', 45); UPDATE sqlite_stat1 SET stat = 123 WHERE tbl='t1' AND idx='i1'; UPDATE sqlite_stat1 SET stat = 456 WHERE tbl='t2'; diff --git a/ext/session/sessionnoop2.test b/ext/session/sessionnoop2.test new file mode 100644 index 0000000000..5549773440 --- /dev/null +++ b/ext/session/sessionnoop2.test @@ -0,0 +1,180 @@ +# 2011 March 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionnoop + +foreach {tn wo} { + 1 "" + 2 " WITHOUT ROWID " +} { + reset_db + eval [string map [list %WO% $wo] { +do_execsql_test $tn.1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c) %WO%; + INSERT INTO t1 VALUES('a', 'A', 'AAA'); + INSERT INTO t1 VALUES('b', 'B', 'BBB'); + INSERT INTO t1 VALUES('c', 'C', 'CCC'); + INSERT INTO t1 VALUES('d', 'D', 'DDD'); + INSERT INTO t1 VALUES('e', 'E', 'EEE'); +} + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_execsql_test -db db2 $tn.1.1 { + CREATE TABLE t1(a PRIMARY KEY, b, c) %WO%; + INSERT INTO t1 VALUES('a', 'A', 'AAA'); + INSERT INTO t1 VALUES('b', 'B', '123'); + INSERT INTO t1 VALUES('c', 'C', 'CCC'); + INSERT INTO t1 VALUES('e', 'E', 'EEE'); + INSERT INTO t1 VALUES('f', 'F', 'FFF'); +} + +set C [changeset_from_sql { + UPDATE t1 SET c='123' WHERE a='b'; + DELETE FROM t1 WHERE a='d'; + INSERT INTO t1 VALUES('f', 'F', 'FFF'); +}] + + +set ::conflict_list [list] +proc xConflict {args} { + lappend ::conflict_list $args + return "OMIT" +} +do_test $tn.1.2 { + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} [list {*}{ + {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 123}} + {INSERT t1 CONFLICT {t f t F t FFF} {t f t F t FFF}} + {DELETE t1 NOTFOUND {t d t D t DDD}} +}] +do_test $tn.1.3 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} [list {*}{ + {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 123}} + {INSERT t1 CONFLICT {t f t F t FFF} {t f t F t FFF}} + {DELETE t1 NOTFOUND {t d t D t DDD}} +}] + +do_test $tn.1.4 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} {} + +do_execsql_test -db db2 1.5 { + UPDATE t1 SET b='G' WHERE a='f'; + UPDATE t1 SET c='456' WHERE a='b'; +} + +do_test $tn.1.6 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} [list {*}{ + {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 456}} + {INSERT t1 CONFLICT {t f t F t FFF} {t f t G t FFF}} +}] + +db2 close + +#-------------------------------------------------------------------------- + +reset_db +forcedelete test.db2 +sqlite3 db2 test.db2 +do_execsql_test $tn.2.0 { + CREATE TABLE t1(a PRIMARY KEY, b) %WO%; +} +do_execsql_test -db db2 $tn.2.1 { + CREATE TABLE t1(a PRIMARY KEY, b, c DEFAULT 'val') %WO%; +} + +do_test $tn.2.2 { + do_then_apply_sql -ignorenoop { + INSERT INTO t1 VALUES(1, 2); + } + do_then_apply_sql -ignorenoop { + UPDATE t1 SET b=2 WHERE a=1 + } +} {} + + +}] +} + +db2 close + +#------------------------------------------------------------------------- +reset_db +forcedelete test.db2 +do_execsql_test 3.0 { + CREATE TABLE xyz(a, b, c, PRIMARY KEY(a, b), UNIQUE(c)); + ANALYZE; + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO xyz SELECT i, i, i FROM s; + VACUUM INTO 'test.db2'; +} + +set C [changeset_from_sql { ANALYZE }] +sqlite3 db2 test.db2 + +set ::conflict_list [list] +proc xConflict {args} { lappend ::conflict_list $args ; return "OMIT" } +do_test 3.1 { + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} {} + +do_test 3.2 { + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} {} + +do_test 3.3 { + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} [list {*}{ + {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}} {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}}} + {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_2 t {100 1}} {t xyz t sqlite_autoindex_xyz_2 t {100 1}}} +}] + +do_execsql_test -db db2 3.4 { + UPDATE sqlite_stat1 SET stat='200 1 1' WHERE idx='sqlite_autoindex_xyz_1'; +} + +do_test 3.5 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} [list {*}{ + {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}} {t xyz t sqlite_autoindex_xyz_1 t {200 1 1}}} +}] + + + +finish_test + diff --git a/ext/session/sessionrebase.test b/ext/session/sessionrebase.test index cdf3322838..033348f9ce 100644 --- a/ext/session/sessionrebase.test +++ b/ext/session/sessionrebase.test @@ -84,6 +84,7 @@ proc do_rebase_test {tn sql1 sql2 conflict_handler {testsql ""} {testres ""}} { db eval BEGIN sqlite3session S1 db main + S1 object_config rowid 1 S1 attach * execsql $sql1 db set c1 [S1 changeset] @@ -91,6 +92,7 @@ proc do_rebase_test {tn sql1 sql2 conflict_handler {testsql ""} {testres ""}} { if {$i==1} { sqlite3session S2 db2 main + S2 object_config rowid 1 S2 attach * execsql $sql2 db2 set c2 [S2 changeset] @@ -100,6 +102,7 @@ proc do_rebase_test {tn sql1 sql2 conflict_handler {testsql ""} {testres ""}} { foreach sql [split $sql2 ";"] { if {[string is space $sql]} continue sqlite3session S2 db2 main + S2 object_config rowid 1 S2 attach * execsql $sql db2 lappend c2 [S2 changeset] @@ -341,6 +344,79 @@ do_rebase_test 2.2.3 { OMIT } { SELECT * FROM t2 WHERE z='B' } { 1 one B } + +reset_db +do_execsql_test 2.3.0 { + CREATE TABLE t1 (b TEXT); + INSERT INTO t1(rowid, b) VALUES(1, 'one'); + INSERT INTO t1(rowid, b) VALUES(2, 'two'); + INSERT INTO t1(rowid, b) VALUES(3, 'three'); +} +do_rebase_test 2.3.1 { + UPDATE t1 SET b = 'two.1' WHERE rowid=2 +} { + UPDATE t1 SET b = 'two.2' WHERE rowid=2; +} { + OMIT +} { SELECT rowid, * FROM t1 } {1 one 2 two.1 3 three} + +do_rebase_test 2.3.2 { + UPDATE t1 SET b = 'two.1' WHERE rowid=2 +} { + UPDATE t1 SET b = 'two.2' WHERE rowid=2; +} { + REPLACE +} { SELECT rowid, * FROM t1 } {1 one 2 two.2 3 three} + +do_rebase_test 2.3.3 { + DELETE FROM t1 WHERE rowid=3 +} { + DELETE FROM t1 WHERE rowid=3; +} { + OMIT +} { SELECT rowid, * FROM t1 } {1 one 2 two} + +do_rebase_test 2.3.4 { + DELETE FROM t1 WHERE rowid=1 +} { + UPDATE t1 SET b='one.2' WHERE rowid=1 +} { + OMIT +} { SELECT rowid, * FROM t1 } {2 two 3 three} + +do_rebase_test 2.3.6 { + UPDATE t1 SET b='three.1' WHERE rowid=3 +} { + DELETE FROM t1 WHERE rowid=3; +} { + OMIT +} { SELECT rowid, * FROM t1 } {1 one 2 two 3 three.1} + +do_rebase_test 2.3.7 { + UPDATE t1 SET b='three.1' WHERE rowid=3 +} { + DELETE FROM t1 WHERE rowid=3; +} { + REPLACE +} { SELECT rowid, * FROM t1 } {1 one 2 two} + +do_rebase_test 2.3.8 { + INSERT INTO t1(rowid, b) VALUES(4, 'four.1') +} { + INSERT INTO t1(rowid, b) VALUES(4, 'four.2'); +} { + REPLACE +} { SELECT rowid, * FROM t1 } {1 one 2 two 3 three 4 four.2} + +do_rebase_test 2.3.9 { + INSERT INTO t1(rowid, b) VALUES(4, 'four.1') +} { + INSERT INTO t1(rowid, b) VALUES(4, 'four.2'); +} { + OMIT +} { SELECT rowid, * FROM t1 } {1 one 2 two 3 three 4 four.1} + + #------------------------------------------------------------------------- reset_db do_execsql_test 3.0 { diff --git a/ext/session/sessionrowid.test b/ext/session/sessionrowid.test new file mode 100644 index 0000000000..14af90acef --- /dev/null +++ b/ext/session/sessionrowid.test @@ -0,0 +1,281 @@ +# 2011 Mar 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the session module. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionrowid + +do_execsql_test 0.0 { + CREATE TABLE t1(a, b); +} + +foreach {tn rowid bEmpty} { + 1 0 1 + 2 1 0 + 3 -1 1 +} { + do_test 0.$tn { + sqlite3session S db main + if {$rowid>=0} { S object_config rowid $rowid } + S attach t1 + execsql { INSERT INTO t1 VALUES(1, 2); } + expr [string length [S changeset]]==0 + } $bEmpty + S delete +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); +} + +do_iterator_test 1.1 t1 { + INSERT INTO t1 VALUES('i', 'one'); +} { + {INSERT t1 0 X.. {} {i 1 t i t one}} +} + +do_execsql_test 1.2 { + SELECT rowid, * FROM t1 +} {1 i one} + +do_iterator_test 1.3 t1 { + UPDATE t1 SET b='two' +} { + {UPDATE t1 0 X.. {i 1 {} {} t one} {{} {} {} {} t two}} +} + +do_iterator_test 1.4 t1 { + DELETE FROM t1; +} { + {DELETE t1 0 X.. {i 1 t i t two} {}} +} + +do_iterator_test 1.5 t1 { + INSERT INTO t1(rowid, a, b) VALUES(14, 'hello', 'world'); + INSERT INTO t1(rowid, a, b) VALUES(NULL, 'yes', 'no'); + INSERT INTO t1(rowid, a, b) VALUES(-123, 'ii', 'iii'); +} { + {INSERT t1 0 X.. {} {i -123 t ii t iii}} + {INSERT t1 0 X.. {} {i 15 t yes t no}} + {INSERT t1 0 X.. {} {i 14 t hello t world}} +} + +do_iterator_test 1.6 t1 { + UPDATE t1 SET a='deluxe' WHERE rowid=14; + DELETE FROM t1 WHERE rowid=-123; + INSERT INTO t1 VALUES('x', 'xi'); +} { + {DELETE t1 0 X.. {i -123 t ii t iii} {}} + {UPDATE t1 0 X.. {i 14 t hello {} {}} {{} {} t deluxe {} {}}} + {INSERT t1 0 X.. {} {i 16 t x t xi}} +} + +#------------------------------------------------------------------------- +reset_db +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_execsql_test 2.0 { + CREATE TABLE t1(a, b); +} +do_execsql_test -db db2 2.0.1 { + CREATE TABLE t1(a, b); +} + +proc xConflict {args} { + puts "CONFLICT!" + return "OMIT" +} + +do_test 2.1 { + set C [changeset_from_sql { + INSERT INTO t1 VALUES('abc', 'def'); + }] + sqlite3changeset_apply db2 $C xConflict + execsql { SELECT * FROM t1 } db2 +} {abc def} +do_test 2.2 { + set C [changeset_from_sql { + UPDATE t1 SET b='hello' + }] + sqlite3changeset_apply db2 $C xConflict + execsql { SELECT * FROM t1 } db2 +} {abc hello} +do_test 2.3 { + set C [changeset_from_sql { + DELETE FROM t1 WHERE b='hello' + }] + sqlite3changeset_apply db2 $C xConflict + execsql { SELECT * FROM t1 } db2 +} {} + +do_test 2.4 { + do_then_apply_sql { + INSERT INTO t1 VALUES('i', 'one'); + INSERT INTO t1 VALUES('ii', 'two'); + INSERT INTO t1 VALUES('iii', 'three'); + INSERT INTO t1 VALUES('iv', 'four'); + } + compare_db db db2 +} {} + +do_test 2.5 { + do_then_apply_sql { + DELETE FROM t1 WHERE a='ii'; + UPDATE t1 SET b='THREE' WHERE a='iii'; + UPDATE t1 SET a='III' WHERE a='iii'; + INSERT INTO t1 VALUES('v', 'five'); + } + compare_db db db2 +} {} + +do_execsql_test 2.6 {SELECT * FROM t1} {i one III THREE iv four v five} +do_execsql_test -db db2 2.7 {SELECT * FROM t1} {i one III THREE iv four v five} + +#------------------------------------------------------------------------- +db2 close +reset_db +forcedelete test.db2 +sqlite3 db2 test.db2 + +set init_sql { + CREATE TABlE t4(a, b); + CREATE INDEX t4a ON t4(a); + CREATE UNIQUE INDEX t4b ON t4(b); +} + +do_execsql_test 3.0 $init_sql +do_execsql_test -db db2 3.0a $init_sql + +do_execsql_test -db db2 3.1 { + INSERT INTO t4(rowid, a, b) VALUES(43, 'hello', 'world'); +} +do_conflict_test 3.2 -sql { + INSERT INTO t4(rowid, a, b) VALUES(43, 'abc', 'def'); +} -tables t4 -conflicts { + {INSERT t4 CONFLICT {i 43 t abc t def} {i 43 t hello t world}} +} +do_execsql_test -db db2 3.3 { + SELECT * FROM t4 +} {hello world} + +do_execsql_test 3.4 { DELETE FROM t4 } +do_conflict_test 3.5 -sql { + INSERT INTO t4(rowid, a, b) VALUES(43, 'abc', 'def'); +} -tables t4 -conflicts { + {INSERT t4 CONFLICT {i 43 t abc t def} {i 43 t hello t world}} +} -policy REPLACE +do_execsql_test -db db2 3.6 { + SELECT * FROM t4 +} {abc def} + +do_execsql_test 3.7 { DELETE FROM t4 } +do_conflict_test 3.8 -sql { + INSERT INTO t4(rowid, a, b) VALUES(45, 'xyz', 'def'); +} -tables t4 -conflicts { + {INSERT t4 CONSTRAINT {i 45 t xyz t def}} +} +do_execsql_test -db db2 3.9 { + SELECT * FROM t4 +} {abc def} + + +do_execsql_test -db db 3.10a { DELETE FROM t4 } +do_execsql_test -db db2 3.10b { DELETE FROM t4 } + +do_execsql_test -db db 3.11a { + INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'one'); + INSERT INTO t4(rowid, a, b) VALUES(222, 'two', 'two'); +} +do_execsql_test -db db2 3.11b { + INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'blip'); +} + +do_conflict_test 3.12 -sql { + DELETE FROM t4 WHERE a='one'; +} -tables t4 -conflicts { + {DELETE t4 DATA {i 111 t one t one} {i 111 t one t blip}} +} +do_execsql_test -db db2 3.13 { + SELECT * FROM t4 +} {one blip} + +do_conflict_test 3.14 -sql { + DELETE FROM t4 WHERE a='two'; +} -tables t4 -conflicts { + {DELETE t4 NOTFOUND {i 222 t two t two}} +} +do_execsql_test -db db2 3.15 { + SELECT * FROM t4 +} {one blip} + +do_execsql_test -db db 3.16a { DELETE FROM t4 } +do_execsql_test -db db2 3.16b { DELETE FROM t4 } + +do_execsql_test -db db 3.17a { + INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'one'); + INSERT INTO t4(rowid, a, b) VALUES(222, 'two', 'two'); +} +do_execsql_test -db db2 3.17b { + INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'blip'); +} + +do_conflict_test 3.18 -sql { + UPDATE t4 SET b='xyz' WHERE a='one' +} -tables t4 -conflicts { + {UPDATE t4 DATA {i 111 {} {} t one} {{} {} {} {} t xyz} {i 111 t one t blip}} +} +do_execsql_test -db db2 3.19 { + SELECT * FROM t4 +} {one blip} + +do_conflict_test 3.20 -sql { + UPDATE t4 SET b='123' WHERE a='two' +} -tables t4 -conflicts { + {UPDATE t4 NOTFOUND {i 222 {} {} t two} {{} {} {} {} t 123}} +} +do_execsql_test -db db2 3.21 { + SELECT * FROM t4 +} {one blip} + +#-------------------------------------------------------------------------- +breakpoint +do_diff_test 4.0 { + CREATE TABLE t1(x, y); + CREATE TABLE aux.t1(x, y); + INSERT INTO t1 VALUES(1, 2); +} + +do_diff_test 4.1 { + CREATE TABLE t1(x, y); + CREATE TABLE aux.t1(x, y); + INSERT INTO aux.t1 VALUES(1, 2); +} + +do_diff_test 4.2 { + CREATE TABLE t1(x, y); + CREATE TABLE aux.t1(x, y); + INSERT INTO t1(rowid, x, y) VALUES(413, 'hello', 'there'); + INSERT INTO aux.t1(rowid, x, y) VALUES(413, 'hello', 'world'); +} + +finish_test + diff --git a/ext/session/sessionsize.test b/ext/session/sessionsize.test index 04d05514db..01638c6677 100644 --- a/ext/session/sessionsize.test +++ b/ext/session/sessionsize.test @@ -113,17 +113,17 @@ do_execsql_test 3.0 { do_test 3.1 { sqlite3session S db main - S object_config_size -1 + S object_config size -1 } 1 -do_test 3.2.1 { S object_config_size 0 } 0 -do_test 3.2.2 { S object_config_size -1 } 0 -do_test 3.2.3 { S object_config_size 1 } 1 -do_test 3.2.4 { S object_config_size -1 } 1 +do_test 3.2.1 { S object_config size 0 } 0 +do_test 3.2.2 { S object_config size -1 } 0 +do_test 3.2.3 { S object_config size 1 } 1 +do_test 3.2.4 { S object_config size -1 } 1 do_test 3.3 { S attach t1 } {} -do_test 3.4 { S object_config_size 1 } {SQLITE_MISUSE} -do_test 3.4 { S object_config_size -1 } {1} +do_test 3.4 { S object_config size 1 } {SQLITE_MISUSE} +do_test 3.4 { S object_config size -1 } {1} S delete diff --git a/ext/session/sessionstat1.test b/ext/session/sessionstat1.test index 774899d96b..2757d60440 100644 --- a/ext/session/sessionstat1.test +++ b/ext/session/sessionstat1.test @@ -82,7 +82,7 @@ do_test 2.0 { } {} do_test 2.1 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { WITH s(i) AS ( SELECT 0 UNION ALL SELECT i+1 FROM s WHERE (i+1)<32 ) @@ -100,7 +100,7 @@ do_execsql_test -db db2 2.2 { } do_test 2.3 { - do_then_apply_sql { DROP INDEX t1c } + do_then_apply_sql -ignorenoop { DROP INDEX t1c } } {} do_execsql_test -db db2 2.4 { @@ -111,7 +111,7 @@ do_execsql_test -db db2 2.4 { } do_test 2.3 { - do_then_apply_sql { DROP TABLE t1 } + do_then_apply_sql -ignorenoop { DROP TABLE t1 } } {} do_execsql_test -db db2 2.4 { @@ -153,16 +153,16 @@ do_execsql_test 3.2 { } {t1 null 4} do_test 3.3 { execsql { DELETE FROM sqlite_stat1 } - do_then_apply_sql { ANALYZE } + do_then_apply_sql -ignorenoop { ANALYZE } execsql { SELECT * FROM sqlite_stat1 } db2 } {t1 null 4} do_test 3.4 { execsql { INSERT INTO t1 VALUES(5,5,5) } - do_then_apply_sql { ANALYZE } + do_then_apply_sql -ignorenoop { ANALYZE } execsql { SELECT * FROM sqlite_stat1 } db2 } {t1 null 5} do_test 3.5 { - do_then_apply_sql { DROP TABLE t1 } + do_then_apply_sql -ignorenoop { DROP TABLE t1 } execsql { SELECT * FROM sqlite_stat1 } db2 } {} diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 4e0ad827eb..1f366a6188 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -25,6 +25,8 @@ typedef struct SessionInput SessionInput; # endif #endif +#define SESSIONS_ROWID "_rowid_" + /* ** The three different types of changesets generated. */ @@ -53,6 +55,7 @@ struct sqlite3_session { int bEnable; /* True if currently recording */ int bIndirect; /* True if all changes are indirect */ int bAutoAttach; /* True to auto-attach tables */ + int bImplicitPK; /* True to handle tables with implicit PK */ int rc; /* Non-zero if an error has occurred */ void *pFilterCtx; /* First argument to pass to xTableFilter */ int (*xTableFilter)(void *pCtx, const char *zTab); @@ -129,6 +132,7 @@ struct SessionTable { char *zName; /* Local name of table */ int nCol; /* Number of columns in table zName */ int bStat1; /* True if this is sqlite_stat1 */ + int bRowid; /* True if this table uses rowid for PK */ const char **azCol; /* Column names */ u8 *abPK; /* Array of primary key flags */ int nEntry; /* Total number of entries in hash table */ @@ -521,6 +525,7 @@ static unsigned int sessionHashAppendType(unsigned int h, int eType){ */ static int sessionPreupdateHash( sqlite3_session *pSession, /* Session object that owns pTab */ + i64 iRowid, SessionTable *pTab, /* Session table handle */ int bNew, /* True to hash the new.* PK */ int *piHash, /* OUT: Hash value */ @@ -529,48 +534,53 @@ static int sessionPreupdateHash( unsigned int h = 0; /* Hash value to return */ int i; /* Used to iterate through columns */ - assert( *pbNullPK==0 ); - assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); - for(i=0; inCol; i++){ - if( pTab->abPK[i] ){ - int rc; - int eType; - sqlite3_value *pVal; + if( pTab->bRowid ){ + assert( pTab->nCol-1==pSession->hook.xCount(pSession->hook.pCtx) ); + h = sessionHashAppendI64(h, iRowid); + }else{ + assert( *pbNullPK==0 ); + assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); + for(i=0; inCol; i++){ + if( pTab->abPK[i] ){ + int rc; + int eType; + sqlite3_value *pVal; - if( bNew ){ - rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); - }else{ - rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); - } - if( rc!=SQLITE_OK ) return rc; + if( bNew ){ + rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); + }else{ + rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); + } + if( rc!=SQLITE_OK ) return rc; - eType = sqlite3_value_type(pVal); - h = sessionHashAppendType(h, eType); - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - i64 iVal; - if( eType==SQLITE_INTEGER ){ - iVal = sqlite3_value_int64(pVal); + eType = sqlite3_value_type(pVal); + h = sessionHashAppendType(h, eType); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_value_int64(pVal); + }else{ + double rVal = sqlite3_value_double(pVal); + assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); + memcpy(&iVal, &rVal, 8); + } + h = sessionHashAppendI64(h, iVal); + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + const u8 *z; + int n; + if( eType==SQLITE_TEXT ){ + z = (const u8 *)sqlite3_value_text(pVal); + }else{ + z = (const u8 *)sqlite3_value_blob(pVal); + } + n = sqlite3_value_bytes(pVal); + if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; + h = sessionHashAppendBlob(h, n, z); }else{ - double rVal = sqlite3_value_double(pVal); - assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); - memcpy(&iVal, &rVal, 8); + assert( eType==SQLITE_NULL ); + assert( pTab->bStat1==0 || i!=1 ); + *pbNullPK = 1; } - h = sessionHashAppendI64(h, iVal); - }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ - const u8 *z; - int n; - if( eType==SQLITE_TEXT ){ - z = (const u8 *)sqlite3_value_text(pVal); - }else{ - z = (const u8 *)sqlite3_value_blob(pVal); - } - n = sqlite3_value_bytes(pVal); - if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; - h = sessionHashAppendBlob(h, n, z); - }else{ - assert( eType==SQLITE_NULL ); - assert( pTab->bStat1==0 || i!=1 ); - *pbNullPK = 1; } } } @@ -853,6 +863,7 @@ static int sessionMergeUpdate( */ static int sessionPreupdateEqual( sqlite3_session *pSession, /* Session object that owns SessionTable */ + i64 iRowid, /* Rowid value if pTab->bRowid */ SessionTable *pTab, /* Table associated with change */ SessionChange *pChange, /* Change to compare to */ int op /* Current pre-update operation */ @@ -860,6 +871,11 @@ static int sessionPreupdateEqual( int iCol; /* Used to iterate through columns */ u8 *a = pChange->aRecord; /* Cursor used to scan change record */ + if( pTab->bRowid ){ + if( a[0]!=SQLITE_INTEGER ) return 0; + return sessionGetI64(&a[1])==iRowid; + } + assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); for(iCol=0; iColnCol; iCol++){ if( !pTab->abPK[iCol] ){ @@ -1004,7 +1020,8 @@ static int sessionTableInfo( int *pnCol, /* OUT: number of columns */ const char **pzTab, /* OUT: Copy of zThis */ const char ***pazCol, /* OUT: Array of column names for table */ - u8 **pabPK /* OUT: Array of booleans - true for PK col */ + u8 **pabPK, /* OUT: Array of booleans - true for PK col */ + int *pbRowid /* OUT: True if only PK is a rowid */ ){ char *zPragma; sqlite3_stmt *pStmt; @@ -1016,6 +1033,7 @@ static int sessionTableInfo( u8 *pAlloc = 0; char **azCol = 0; u8 *abPK = 0; + int bRowid = 0; /* Set to true to use rowid as PK */ assert( pazCol && pabPK ); @@ -1060,10 +1078,15 @@ static int sessionTableInfo( } nByte = nThis + 1; + bRowid = (pbRowid!=0); while( SQLITE_ROW==sqlite3_step(pStmt) ){ nByte += sqlite3_column_bytes(pStmt, 1); nDbCol++; + if( sqlite3_column_int(pStmt, 5) ) bRowid = 0; } + if( nDbCol==0 ) bRowid = 0; + nDbCol += bRowid; + nByte += strlen(SESSIONS_ROWID); rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ){ @@ -1085,6 +1108,14 @@ static int sessionTableInfo( } i = 0; + if( bRowid ){ + size_t nName = strlen(SESSIONS_ROWID); + memcpy(pAlloc, SESSIONS_ROWID, nName+1); + azCol[i] = (char*)pAlloc; + pAlloc += nName+1; + abPK[i] = 1; + i++; + } while( SQLITE_ROW==sqlite3_step(pStmt) ){ int nName = sqlite3_column_bytes(pStmt, 1); const unsigned char *zName = sqlite3_column_text(pStmt, 1); @@ -1096,7 +1127,6 @@ static int sessionTableInfo( i++; } rc = sqlite3_reset(pStmt); - } /* If successful, populate the output variables. Otherwise, zero them and @@ -1113,6 +1143,7 @@ static int sessionTableInfo( if( pzTab ) *pzTab = 0; sessionFree(pSession, azCol); } + if( pbRowid ) *pbRowid = bRowid; sqlite3_finalize(pStmt); return rc; } @@ -1134,7 +1165,8 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ u8 *abPK; assert( pTab->azCol==0 || pTab->abPK==0 ); pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, - pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK + pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK, + (pSession->bImplicitPK ? &pTab->bRowid : 0) ); if( pSession->rc==SQLITE_OK ){ int i; @@ -1206,6 +1238,7 @@ static int sessionUpdateMaxSize( ){ i64 nNew = 2; if( pC->op==SQLITE_INSERT ){ + if( pTab->bRowid ) nNew += 9; if( op!=SQLITE_DELETE ){ int ii; for(ii=0; iinCol; ii++){ @@ -1222,12 +1255,16 @@ static int sessionUpdateMaxSize( }else{ int ii; u8 *pCsr = pC->aRecord; - for(ii=0; iinCol; ii++){ + if( pTab->bRowid ){ + nNew += 9 + 1; + pCsr += 9; + } + for(ii=pTab->bRowid; iinCol; ii++){ int bChanged = 1; int nOld = 0; int eType; sqlite3_value *p = 0; - pSession->hook.xNew(pSession->hook.pCtx, ii, &p); + pSession->hook.xNew(pSession->hook.pCtx, ii-pTab->bRowid, &p); if( p==0 ){ return SQLITE_NOMEM; } @@ -1306,6 +1343,7 @@ static int sessionUpdateMaxSize( */ static void sessionPreupdateOneChange( int op, /* One of SQLITE_UPDATE, INSERT, DELETE */ + i64 iRowid, sqlite3_session *pSession, /* Session object pTab is attached to */ SessionTable *pTab /* Table that change applies to */ ){ @@ -1321,7 +1359,7 @@ static void sessionPreupdateOneChange( /* Check the number of columns in this xPreUpdate call matches the ** number of columns in the table. */ - if( pTab->nCol!=pSession->hook.xCount(pSession->hook.pCtx) ){ + if( (pTab->nCol-pTab->bRowid)!=pSession->hook.xCount(pSession->hook.pCtx) ){ pSession->rc = SQLITE_SCHEMA; return; } @@ -1354,14 +1392,16 @@ static void sessionPreupdateOneChange( /* Calculate the hash-key for this change. If the primary key of the row ** includes a NULL value, exit early. Such changes are ignored by the ** session module. */ - rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull); + rc = sessionPreupdateHash( + pSession, iRowid, pTab, op==SQLITE_INSERT, &iHash, &bNull + ); if( rc!=SQLITE_OK ) goto error_out; if( bNull==0 ){ /* Search the hash table for an existing record for this row. */ SessionChange *pC; for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){ - if( sessionPreupdateEqual(pSession, pTab, pC, op) ) break; + if( sessionPreupdateEqual(pSession, iRowid, pTab, pC, op) ) break; } if( pC==0 ){ @@ -1376,7 +1416,7 @@ static void sessionPreupdateOneChange( /* Figure out how large an allocation is required */ nByte = sizeof(SessionChange); - for(i=0; inCol; i++){ + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p); @@ -1391,6 +1431,9 @@ static void sessionPreupdateOneChange( rc = sessionSerializeValue(0, p, &nByte); if( rc!=SQLITE_OK ) goto error_out; } + if( pTab->bRowid ){ + nByte += 9; /* Size of rowid field - an integer */ + } /* Allocate the change object */ pC = (SessionChange *)sessionMalloc64(pSession, nByte); @@ -1407,7 +1450,12 @@ static void sessionPreupdateOneChange( ** required values and encodings have already been cached in memory. ** It is not possible for an OOM to occur in this block. */ nByte = 0; - for(i=0; inCol; i++){ + if( pTab->bRowid ){ + pC->aRecord[0] = SQLITE_INTEGER; + sessionPutI64(&pC->aRecord[1], iRowid); + nByte = 9; + } + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ pSession->hook.xOld(pSession->hook.pCtx, i, &p); @@ -1522,9 +1570,10 @@ static void xPreUpdate( pSession->rc = sessionFindTable(pSession, zName, &pTab); if( pTab ){ assert( pSession->rc==SQLITE_OK ); - sessionPreupdateOneChange(op, pSession, pTab); + assert( op==SQLITE_UPDATE || iKey1==iKey2 ); + sessionPreupdateOneChange(op, iKey1, pSession, pTab); if( op==SQLITE_UPDATE ){ - sessionPreupdateOneChange(SQLITE_INSERT, pSession, pTab); + sessionPreupdateOneChange(SQLITE_INSERT, iKey2, pSession, pTab); } } } @@ -1563,6 +1612,7 @@ static void sessionPreupdateHooks( typedef struct SessionDiffCtx SessionDiffCtx; struct SessionDiffCtx { sqlite3_stmt *pStmt; + int bRowid; int nOldOff; }; @@ -1571,17 +1621,17 @@ struct SessionDiffCtx { */ static int sessionDiffOld(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff+p->bRowid); return SQLITE_OK; } static int sessionDiffNew(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->bRowid); return SQLITE_OK; } static int sessionDiffCount(void *pCtx){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt); + return (p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt)) - p->bRowid; } static int sessionDiffDepth(void *pCtx){ (void)pCtx; @@ -1660,14 +1710,16 @@ static char *sessionExprCompareOther( static char *sessionSelectFindNew( const char *zDb1, /* Pick rows in this db only */ const char *zDb2, /* But not in this one */ + int bRowid, const char *zTbl, /* Table name */ const char *zExpr ){ + const char *zSel = (bRowid ? SESSIONS_ROWID ", *" : "*"); char *zRet = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\" WHERE NOT EXISTS (" + "SELECT %s FROM \"%w\".\"%w\" WHERE NOT EXISTS (" " SELECT 1 FROM \"%w\".\"%w\" WHERE %s" ")", - zDb1, zTbl, zDb2, zTbl, zExpr + zSel, zDb1, zTbl, zDb2, zTbl, zExpr ); return zRet; } @@ -1681,7 +1733,9 @@ static int sessionDiffFindNew( char *zExpr ){ int rc = SQLITE_OK; - char *zStmt = sessionSelectFindNew(zDb1, zDb2, pTab->zName,zExpr); + char *zStmt = sessionSelectFindNew( + zDb1, zDb2, pTab->bRowid, pTab->zName, zExpr + ); if( zStmt==0 ){ rc = SQLITE_NOMEM; @@ -1692,8 +1746,10 @@ static int sessionDiffFindNew( SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx; pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = 0; + pDiffCtx->bRowid = pTab->bRowid; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(op, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(op, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } @@ -1703,6 +1759,27 @@ static int sessionDiffFindNew( return rc; } +/* +** Return a comma-separated list of the fully-qualified (with both database +** and table name) column names from table pTab. e.g. +** +** "main"."t1"."a", "main"."t1"."b", "main"."t1"."c" +*/ +static char *sessionAllCols( + const char *zDb, + SessionTable *pTab +){ + int ii; + char *zRet = 0; + for(ii=0; iinCol; ii++){ + zRet = sqlite3_mprintf("%z%s\"%w\".\"%w\".\"%w\"", + zRet, (zRet ? ", " : ""), zDb, pTab->zName, pTab->azCol[ii] + ); + if( !zRet ) break; + } + return zRet; +} + static int sessionDiffFindModified( sqlite3_session *pSession, SessionTable *pTab, @@ -1717,11 +1794,13 @@ static int sessionDiffFindModified( if( zExpr2==0 ){ rc = SQLITE_NOMEM; }else{ + char *z1 = sessionAllCols(pSession->zDb, pTab); + char *z2 = sessionAllCols(zFrom, pTab); char *zStmt = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", - pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 + "SELECT %s,%s FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", + z1, z2, pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 ); - if( zStmt==0 ){ + if( zStmt==0 || z1==0 || z2==0 ){ rc = SQLITE_NOMEM; }else{ sqlite3_stmt *pStmt; @@ -1732,12 +1811,15 @@ static int sessionDiffFindModified( pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = pTab->nCol; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(SQLITE_UPDATE, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(SQLITE_UPDATE, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } - sqlite3_free(zStmt); } + sqlite3_free(zStmt); + sqlite3_free(z1); + sqlite3_free(z2); } return rc; @@ -1776,9 +1858,12 @@ int sqlite3session_diff( int bHasPk = 0; int bMismatch = 0; int nCol; /* Columns in zFrom.zTbl */ + int bRowid = 0; u8 *abPK; const char **azCol = 0; - rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK); + rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK, + pSession->bImplicitPK ? &bRowid : 0 + ); if( rc==SQLITE_OK ){ if( pTo->nCol!=nCol ){ bMismatch = 1; @@ -2120,9 +2205,10 @@ static void sessionAppendStr( int *pRc ){ int nStr = sqlite3Strlen30(zStr); - if( 0==sessionBufferGrow(p, nStr, pRc) ){ + if( 0==sessionBufferGrow(p, nStr+1, pRc) ){ memcpy(&p->aBuf[p->nBuf], zStr, nStr); p->nBuf += nStr; + p->aBuf[p->nBuf] = 0x00; } } @@ -2144,6 +2230,27 @@ static void sessionAppendInteger( sessionAppendStr(p, aBuf, pRc); } +static void sessionAppendPrintf( + SessionBuffer *p, /* Buffer to append to */ + int *pRc, + const char *zFmt, + ... +){ + if( *pRc==SQLITE_OK ){ + char *zApp = 0; + va_list ap; + va_start(ap, zFmt); + zApp = sqlite3_vmprintf(zFmt, ap); + if( zApp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sessionAppendStr(p, zApp, pRc); + } + va_end(ap); + sqlite3_free(zApp); + } +} + /* ** This function is a no-op if *pRc is other than SQLITE_OK when it is ** called. Otherwise, append the string zStr enclosed in quotes (") and @@ -2158,7 +2265,7 @@ static void sessionAppendIdent( const char *zStr, /* String to quote, escape and append */ int *pRc /* IN/OUT: Error code */ ){ - int nStr = sqlite3Strlen30(zStr)*2 + 2 + 1; + int nStr = sqlite3Strlen30(zStr)*2 + 2 + 2; if( 0==sessionBufferGrow(p, nStr, pRc) ){ char *zOut = (char *)&p->aBuf[p->nBuf]; const char *zIn = zStr; @@ -2169,6 +2276,7 @@ static void sessionAppendIdent( } *zOut++ = '"'; p->nBuf = (int)((u8 *)zOut - p->aBuf); + p->aBuf[p->nBuf] = 0x00; } } @@ -2304,7 +2412,7 @@ static int sessionAppendUpdate( /* If at least one field has been modified, this is not a no-op. */ if( bChanged ) bNoop = 0; - /* Add a field to the old.* record. This is omitted if this modules is + /* Add a field to the old.* record. This is omitted if this module is ** currently generating a patchset. */ if( ePatchset!=SESSIONS_PATCHSET ){ if( ePatchset==SESSIONS_FULLCHANGESET || bChanged || abPK[i] ){ @@ -2393,12 +2501,20 @@ static int sessionAppendDelete( ** Formulate and prepare a SELECT statement to retrieve a row from table ** zTab in database zDb based on its primary key. i.e. ** -** SELECT * FROM zDb.zTab WHERE pk1 = ? AND pk2 = ? AND ... +** SELECT *, FROM zDb.zTab WHERE (pk1, pk2,...) IS (?1, ?2,...) +** +** where is: +** +** 1 AND (?A OR ?1 IS ) AND ... +** +** for each non-pk . */ static int sessionSelectStmt( sqlite3 *db, /* Database handle */ + int bIgnoreNoop, const char *zDb, /* Database name */ const char *zTab, /* Table name */ + int bRowid, int nCol, /* Number of columns in table */ const char **azCol, /* Names of table columns */ u8 *abPK, /* PRIMARY KEY array */ @@ -2406,8 +2522,50 @@ static int sessionSelectStmt( ){ int rc = SQLITE_OK; char *zSql = 0; + const char *zSep = ""; + const char *zCols = bRowid ? SESSIONS_ROWID ", *" : "*"; int nSql = -1; + int i; + SessionBuffer nooptest = {0, 0, 0}; + SessionBuffer pkfield = {0, 0, 0}; + SessionBuffer pkvar = {0, 0, 0}; + + sessionAppendStr(&nooptest, ", 1", &rc); + + if( 0==sqlite3_stricmp("sqlite_stat1", zTab) ){ + sessionAppendStr(&nooptest, " AND (?6 OR ?3 IS stat)", &rc); + sessionAppendStr(&pkfield, "tbl, idx", &rc); + sessionAppendStr(&pkvar, + "?1, (CASE WHEN ?2=X'' THEN NULL ELSE ?2 END)", &rc + ); + zCols = "tbl, ?2, stat"; + }else{ + for(i=0; izDb, zName, &nCol, 0,&azCol,&abPK); - if( !rc && (pTab->nCol!=nCol || memcmp(abPK, pTab->abPK, nCol)) ){ + rc = sessionTableInfo( + 0, db, pSession->zDb, zName, &nCol, 0, &azCol, &abPK, + (pSession->bImplicitPK ? &bRowid : 0) + ); + if( rc==SQLITE_OK && ( + pTab->nCol!=nCol + || pTab->bRowid!=bRowid + || memcmp(abPK, pTab->abPK, nCol) + )){ rc = SQLITE_SCHEMA; } @@ -2600,7 +2769,8 @@ static int sessionGenerateChangeset( /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ rc = sessionSelectStmt( - db, pSession->zDb, zName, nCol, azCol, abPK, &pSel); + db, 0, pSession->zDb, zName, bRowid, nCol, azCol, abPK, &pSel + ); } nNoop = buf.nBuf; @@ -2815,6 +2985,19 @@ int sqlite3session_object_config(sqlite3_session *pSession, int op, void *pArg){ break; } + case SQLITE_SESSION_OBJCONFIG_ROWID: { + int iArg = *(int*)pArg; + if( iArg>=0 ){ + if( pSession->pTable ){ + rc = SQLITE_MISUSE; + }else{ + pSession->bImplicitPK = (iArg!=0); + } + } + *(int*)pArg = pSession->bImplicitPK; + break; + } + default: rc = SQLITE_MISUSE; } @@ -3803,6 +3986,8 @@ struct SessionApplyCtx { SessionBuffer rebase; /* Rebase information (if any) here */ u8 bRebaseStarted; /* If table header is already in rebase */ u8 bRebase; /* True to collect rebase information */ + u8 bIgnoreNoop; /* True to ignore no-op conflicts */ + int bRowid; }; /* Number of prepared UPDATE statements to cache. */ @@ -4053,8 +4238,10 @@ static int sessionSelectRow( const char *zTab, /* Table name */ SessionApplyCtx *p /* Session changeset-apply context */ ){ - return sessionSelectStmt( - db, "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect); + /* TODO */ + return sessionSelectStmt(db, p->bIgnoreNoop, + "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect + ); } /* @@ -4213,20 +4400,33 @@ static int sessionBindRow( */ static int sessionSeekToRow( sqlite3_changeset_iter *pIter, /* Changeset iterator */ - u8 *abPK, /* Primary key flags array */ - sqlite3_stmt *pSelect /* SELECT statement from sessionSelectRow() */ + SessionApplyCtx *p ){ + sqlite3_stmt *pSelect = p->pSelect; int rc; /* Return code */ int nCol; /* Number of columns in table */ int op; /* Changset operation (SQLITE_UPDATE etc.) */ const char *zDummy; /* Unused */ + sqlite3_clear_bindings(pSelect); sqlite3changeset_op(pIter, &zDummy, &nCol, &op, 0); rc = sessionBindRow(pIter, op==SQLITE_INSERT ? sqlite3changeset_new : sqlite3changeset_old, - nCol, abPK, pSelect + nCol, p->abPK, pSelect ); + if( op!=SQLITE_DELETE && p->bIgnoreNoop ){ + int ii; + for(ii=0; rc==SQLITE_OK && iiabPK[ii]==0 ){ + sqlite3_value *pVal = 0; + sqlite3changeset_new(pIter, ii, &pVal); + sqlite3_bind_int(pSelect, ii+1+nCol, (pVal==0)); + if( pVal ) rc = sessionBindValue(pSelect, ii+1, pVal); + } + } + } + if( rc==SQLITE_OK ){ rc = sqlite3_step(pSelect); if( rc!=SQLITE_ROW ) rc = sqlite3_reset(pSelect); @@ -4341,16 +4541,22 @@ static int sessionConflictHandler( /* Bind the new.* PRIMARY KEY values to the SELECT statement. */ if( pbReplace ){ - rc = sessionSeekToRow(pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); }else{ rc = SQLITE_OK; } if( rc==SQLITE_ROW ){ /* There exists another row with the new.* primary key. */ - pIter->pConflict = p->pSelect; - res = xConflict(pCtx, eType, pIter); - pIter->pConflict = 0; + if( p->bIgnoreNoop + && sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1) + ){ + res = SQLITE_CHANGESET_OMIT; + }else{ + pIter->pConflict = p->pSelect; + res = xConflict(pCtx, eType, pIter); + pIter->pConflict = 0; + } rc = sqlite3_reset(p->pSelect); }else if( rc==SQLITE_OK ){ if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){ @@ -4458,7 +4664,7 @@ static int sessionApplyOneOp( sqlite3_step(p->pDelete); rc = sqlite3_reset(p->pDelete); - if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 && p->bIgnoreNoop==0 ){ rc = sessionConflictHandler( SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry ); @@ -4515,7 +4721,7 @@ static int sessionApplyOneOp( /* Check if there is a conflicting row. For sqlite_stat1, this needs ** to be done using a SELECT, as there is no PRIMARY KEY in the ** database schema to throw an exception if a duplicate is inserted. */ - rc = sessionSeekToRow(pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); if( rc==SQLITE_ROW ){ rc = SQLITE_CONSTRAINT; sqlite3_reset(p->pSelect); @@ -4692,6 +4898,7 @@ static int sessionChangesetApply( memset(&sApply, 0, sizeof(sApply)); sApply.bRebase = (ppRebase && pnRebase); sApply.bInvertConstraints = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + sApply.bIgnoreNoop = !!(flags & SQLITE_CHANGESETAPPLY_IGNORENOOP); sqlite3_mutex_enter(sqlite3_db_mutex(db)); if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){ rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); @@ -4729,6 +4936,7 @@ static int sessionChangesetApply( sApply.bStat1 = 0; sApply.bDeferConstraints = 1; sApply.bRebaseStarted = 0; + sApply.bRowid = 0; memset(&sApply.constraints, 0, sizeof(SessionBuffer)); /* If an xFilter() callback was specified, invoke it now. If the @@ -4748,8 +4956,8 @@ static int sessionChangesetApply( int i; sqlite3changeset_pk(pIter, &abPK, 0); - rc = sessionTableInfo(0, - db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK + rc = sessionTableInfo(0, db, "main", zNew, + &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK, &sApply.bRowid ); if( rc!=SQLITE_OK ) break; for(i=0; iSQLITE_SESSION_OBJCONFIG_SIZE
@@ -105,12 +109,21 @@ void sqlite3session_delete(sqlite3_session *pSession); ** ** It is an error (SQLITE_MISUSE) to attempt to modify this setting after ** the first table has been attached to the session object. +** +**
SQLITE_SESSION_OBJCONFIG_ROWID
+** This option is used to set, clear or query the flag that enables +** collection of data for tables with no explicit PRIMARY KEY. +** +** Normally, tables with no explicit PRIMARY KEY are simply ignored +** by the sessions module. However, if this flag is set, it behaves +** as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted +** as their leftmost columns. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. */ -int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); - -/* -*/ -#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_ROWID 2 /* ** CAPI3REF: Enable Or Disable A Session Object @@ -1256,9 +1269,23 @@ int sqlite3changeset_apply_v2( ** Invert the changeset before applying it. This is equivalent to inverting ** a changeset using sqlite3changeset_invert() before applying it. It is ** an error to specify this flag with a patchset. +** +**
SQLITE_CHANGESETAPPLY_IGNORENOOP
+** Do not invoke the conflict handler callback for any changes that +** would not actually modify the database even if they were applied. +** Specifically, this means that the conflict handler is not invoked +** for: +**
    +**
  • a delete change if the row being deleted cannot be found, +**
  • an update change if the modified fields are already set to +** their new values in the conflicting row, or +**
  • an insert change if all fields of the conflicting row match +** the row being inserted. +**
*/ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 #define SQLITE_CHANGESETAPPLY_INVERT 0x0002 +#define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 /* ** CAPI3REF: Constants Passed To The Conflict Handler diff --git a/ext/session/test_session.c b/ext/session/test_session.c index f8706fb1c3..754fe9cfe1 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -76,9 +76,11 @@ int sql_exec_changeset( ){ sqlite3_session *pSession = 0; int rc; + int val = 1; /* Create a new session object */ rc = sqlite3session_create(db, "main", &pSession); + sqlite3session_object_config(pSession, SQLITE_SESSION_OBJCONFIG_ROWID, &val); /* Configure the session object to record changes to all tables */ if( rc==SQLITE_OK ) rc = sqlite3session_attach(pSession, NULL); @@ -261,7 +263,7 @@ static int SQLITE_TCLAPI test_session_cmd( { "fullchangeset",0, "" }, /* 9 */ { "memory_used", 0, "", }, /* 10 */ { "changeset_size", 0, "", }, /* 11 */ - { "object_config_size", 1, "INTEGER", }, /* 12 */ + { "object_config", 2, "OPTION INTEGER", }, /* 12 */ { 0 } }; int iSub; @@ -384,15 +386,26 @@ static int SQLITE_TCLAPI test_session_cmd( Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize)); break; } - case 12: { + case 12: { /* object_config */ + struct ObjConfOpt { + const char *zName; + int opt; + } aOpt[] = { + { "size", SQLITE_SESSION_OBJCONFIG_SIZE }, + { "rowid", SQLITE_SESSION_OBJCONFIG_ROWID }, + { 0, 0 } + }; + size_t sz = sizeof(aOpt[0]); int rc; int iArg; - if( Tcl_GetIntFromObj(interp, objv[2], &iArg) ){ + int iOpt; + if( Tcl_GetIndexFromObjStruct(interp,objv[2],aOpt,sz,"option",0,&iOpt) ){ return TCL_ERROR; } - rc = sqlite3session_object_config( - pSession, SQLITE_SESSION_OBJCONFIG_SIZE, &iArg - ); + if( Tcl_GetIntFromObj(interp, objv[3], &iArg) ){ + return TCL_ERROR; + } + rc = sqlite3session_object_config(pSession, aOpt[iOpt].opt, &iArg); if( rc!=SQLITE_OK ){ extern const char *sqlite3ErrName(int); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); @@ -798,32 +811,31 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( memset(&sStr, 0, sizeof(sStr)); sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); - /* Check for the -nosavepoint flag */ + /* Check for the -nosavepoint, -invert or -ignorenoop switches */ if( bV2 ){ - if( objc>1 ){ + while( objc>1 ){ const char *z1 = Tcl_GetString(objv[1]); int n = strlen(z1); if( n>1 && n<=12 && 0==sqlite3_strnicmp("-nosavepoint", z1, n) ){ flags |= SQLITE_CHANGESETAPPLY_NOSAVEPOINT; - objc--; - objv++; } - } - if( objc>1 ){ - const char *z1 = Tcl_GetString(objv[1]); - int n = strlen(z1); - if( n>1 && n<=7 && 0==sqlite3_strnicmp("-invert", z1, n) ){ + else if( n>2 && n<=7 && 0==sqlite3_strnicmp("-invert", z1, n) ){ flags |= SQLITE_CHANGESETAPPLY_INVERT; - objc--; - objv++; } + else if( n>2 && n<=11 && 0==sqlite3_strnicmp("-ignorenoop", z1, n) ){ + flags |= SQLITE_CHANGESETAPPLY_IGNORENOOP; + }else{ + break; + } + objc--; + objv++; } } if( objc!=4 && objc!=5 ){ const char *zMsg; if( bV2 ){ - zMsg = "?-nosavepoint? ?-inverse? " + zMsg = "?-nosavepoint? ?-inverse? ?-ignorenoop? " "DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?"; }else{ zMsg = "DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?"; diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 7ffd866f24..a99513bfa8 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -44,7 +44,7 @@ # 1) Consolidate the code generation for sqlite3*.*js into a script # which generates the makefile code, rather than using $(call) and # $(eval), or at least centralize the setup of the numerous vars -# related to each build variant (vanilla, esm, bundler-friendly). +# related to each build variant $(JS_BUILD_MODES). # SHELL := $(shell which bash 2>/dev/null) MAKEFILE := $(lastword $(MAKEFILE_LIST)) @@ -52,7 +52,9 @@ CLEAN_FILES := DISTCLEAN_FILES := ./--dummy-- default: all release: oz - +# JS_BUILD_MODES exists solely to reduce repetition in documentation +# below. +JS_BUILD_MODES := vanilla esm bunder-friendly node # Emscripten SDK home dir and related binaries... EMSDK_HOME ?= $(word 1,$(wildcard $(HOME)/emsdk $(HOME)/src/emsdk)) emcc.bin ?= $(word 1,$(wildcard $(EMSDK_HOME)/upstream/emscripten/emcc) $(shell which emcc)) @@ -145,8 +147,27 @@ ifeq (,$(wildcard $(dir.tmp))) dir._tmp := $(shell mkdir -p $(dir.tmp)) endif -sqlite3.c := $(dir.top)/sqlite3.c +######################################################################## +# Set up sqlite3.c and sqlite3.h... +# +# To build with SEE (https://sqlite.org/see), either put sqlite3-see.c +# in the top of this build tree or pass +# sqlite3.c=PATH_TO_sqlite3-see.c to the build. Note that only +# encryption modules with no 3rd-party dependencies will currently +# work here: AES256-OFB, AES128-OFB, and AES128-CCM. Not +# coincidentally, those 3 modules are included in the sqlite3-see.c +# bundle. +# +# A custom sqlite3.c must not have any spaces in its name. +sqlite3.canonical.c := $(dir.top)/sqlite3.c +sqlite3.c ?= $(firstword $(wildcard $(dir.top)/sqlite3-see.c) $(sqlite3.canonical.c)) sqlite3.h := $(dir.top)/sqlite3.h +ifeq (,$(shell grep sqlite3_activate_see $(sqlite3.c) 2>/dev/null)) + SQLITE_C_IS_SEE := 0 +else + SQLITE_C_IS_SEE := 1 + $(info This is an SEE build.) +endif # Most SQLITE_OPT flags are set in sqlite3-wasm.c but we need them # made explicit here for building speedtest1.c. SQLITE_OPT = \ @@ -169,10 +190,13 @@ SQLITE_OPT = \ -DSQLITE_OS_KV_OPTIONAL=1 \ '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ -DSQLITE_USE_URI=1 \ - -DSQLITE_WASM_ENABLE_C_TESTS + -DSQLITE_WASM_ENABLE_C_TESTS \ + -DSQLITE_C=$(sqlite3.c) -$(sqlite3.c) $(sqlite3.h): +.NOTPARALLEL: $(sqlite3.h) +$(sqlite3.h): $(MAKE) -C $(dir.top) sqlite3.c +$(sqlite3.c): $(sqlite3.h) .PHONY: clean distclean clean: @@ -189,13 +213,42 @@ else $(info Development build. Use '$(MAKE) release' for a smaller release build.) endif +######################################################################## +# Adding custom C code via sqlite3_wasm_extra_init.c: +# +# If the canonical build process finds the file +# sqlite3_wasm_extra_init.c in the main wasm build directory, it +# arranges to include that file in the build of sqlite3.wasm and +# defines SQLITE_EXTRA_INIT=sqlite3_wasm_extra_init. +# +# sqlite3_wasm_extra_init() must be a function with this signature: +# +# int sqlite3_wasm_extra_init(const char *) +# +# and the sqlite3 library will call it with an argument of NULL one +# time during sqlite3_initialize(). If it returns non-0, +# initialization of the library will fail. +# +# The filename can be overridden with: +# +# make sqlite3_wasm_extra_init.c=my_custom_stuff.c +# +# See example_extra_init.c for an example implementation. +######################################################################## +sqlite3_wasm_extra_init.c ?= $(wildcard sqlite3_wasm_extra_init.c) +cflags.wasm_extra_init := +ifneq (,$(sqlite3_wasm_extra_init.c)) + $(info Enabling SQLITE_EXTRA_INIT via $(sqlite3_wasm_extra_init.c).) + cflags.wasm_extra_init := -DSQLITE_WASM_EXTRA_INIT +endif + # bin.version-info = binary to output various sqlite3 version info for # embedding in the JS files and in building the distribution zip file. # It must NOT be in $(dir.tmp) because we need it to survive the # cleanup process for the dist build to work properly. bin.version-info := $(dir.wasm)/version-info $(bin.version-info): $(dir.wasm)/version-info.c $(sqlite3.h) $(MAKEFILE) - $(CC) -O0 -I$(dir.top) -o $@ $< + $(CC) -O0 -I$(dir $(sqlite3.c)) -o $@ $< DISTCLEAN_FILES += $(bin.version-info) # bin.stripcomments is used for stripping C/C++-style comments from JS @@ -253,7 +306,7 @@ endef ######################################################################## # cflags.common = C compiler flags for all builds -cflags.common := -I. -I.. -I$(dir.top) +cflags.common := -I. -I$(dir $(sqlite3.c)) # emcc.WASM_BIGINT = 1 for BigInt (C int64) support, else 0. The API # disables certain features if BigInt is not enabled and such builds # _are not tested_ on any regular basis. @@ -296,10 +349,14 @@ emcc_opt_full := $(emcc_opt) -g3 # EXPORTED_FUNCTIONS.* = files for use with Emscripten's # -sEXPORTED_FUNCTION flag. -EXPORTED_FUNCTIONS.api.in := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) +EXPORTED_FUNCTIONS.api.main := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) +EXPORTED_FUNCTIONS.api.in := $(EXPORTED_FUNCTIONS.api.main) +ifeq (1,$(SQLITE_C_IS_SEE)) + EXPORTED_FUNCTIONS.api.in += $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see) +endif EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api -$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) - cp $(EXPORTED_FUNCTIONS.api.in) $@ +$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) + cat $(EXPORTED_FUNCTIONS.api.in) > $@ # sqlite3-license-version.js = generated JS file with the license # header and version info. @@ -402,14 +459,20 @@ emcc.exportedRuntimeMethods := \ emcc.jsflags += $(emcc.exportedRuntimeMethods) emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 emcc.jsflags += -sIMPORTED_MEMORY -ifeq (3.1.31,$(emcc.version)) - emcc.jsflags += -sSTRICT_JS=0 - $(warning Disabling -sSTRICT_JS for emcc $(emcc.version): \ - https://github.com/emscripten-core/emscripten/issues/18610) -else - emcc.jsflags += -sSTRICT_JS=1 -endif -emcc.environment := -sENVIRONMENT=web,worker +emcc.jsflags += -sSTRICT_JS=0 +# STRICT_JS disabled due to: +# https://github.com/emscripten-core/emscripten/issues/18610 +# TL;DR: does not work with MODULARIZE or EXPORT_ES6 as of version 3.1.31. + +# -sENVIRONMENT values for the various build modes: +emcc.environment.vanilla := web,worker +emcc.environment.bundler-friendly := $(emcc.environment.vanilla) +emcc.environment.esm := $(emcc.environment.vanilla) +emcc.environment.node := node +# Note that adding "node" to the list for the other builds causes +# Emscripten to generate code which confuses node: it cannot reliably +# determine whether the build is for a browser or for node. + ######################################################################## # -sINITIAL_MEMORY: How much memory we need to start with is governed # at least in part by whether -sALLOW_MEMORY_GROWTH is enabled. If so, @@ -422,9 +485,9 @@ emcc.environment := -sENVIRONMENT=web,worker # such test results are inconsistent due to browser internals which # are opaque to us. emcc.jsflags += -sALLOW_MEMORY_GROWTH -emcc.INITIAL_MEMORY.128 := 13107200 +emcc.INITIAL_MEMORY.128 := 134217728 emcc.INITIAL_MEMORY.96 := 100663296 -emcc.INITIAL_MEMORY.64 := 64225280 +emcc.INITIAL_MEMORY.64 := 67108864 emcc.INITIAL_MEMORY.32 := 33554432 emcc.INITIAL_MEMORY.16 := 16777216 emcc.INITIAL_MEMORY.8 := 8388608 @@ -441,6 +504,8 @@ emcc.jsflags += -sSTACK_SIZE=512KB # ^^^ ACHTUNG: emsdk 3.1.27 reduced the default stack size from 5MB to # a mere 64KB, which leads to silent memory corruption via the kvvfs # VFS, which requires twice that for its xRead() and xWrite() methods. +# 2023-03: those methods have since been adapted to use a malloc()'d +# buffer. ######################################################################## # $(sqlite3.js.init-func) is the name Emscripten assigns our exported # module init/load function. This symbol name is hard-coded in @@ -509,7 +574,7 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) @echo "Making $@..." @{ \ - echo 'self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){'; \ + echo 'globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){'; \ echo -n ' sqlite3.version = '; \ $(bin.version-info) --json; \ echo ';'; \ @@ -551,10 +616,11 @@ $(post-js.js.in): $(post-jses.js) $(MAKEFILE) ######################################################################## # call-make-pre-post is a $(call)able which creates rules for # pre-js-$(1).js. $1 = the base name of the JS file on whose behalf -# this pre-js is for (one of: sqlite3, sqlite3-wasmfs). $2 is the build -# mode: one of (vanilla, esm, bundler-friendly). This sets up -# --[extern-][pre/post]-js flags in $(pre-post-$(1).flags.$(2)) and -# dependencies in $(pre-post-$(1).deps.$(2)). +# this pre-js is for (one of: sqlite3, sqlite3-wasmfs). $2 is the +# build mode: one of $(JS_BUILD_MODES). This +# sets up --[extern-][pre/post]-js flags in +# $(pre-post-$(1).flags.$(2)) and dependencies in +# $(pre-post-$(1).deps.$(2)). define call-make-pre-post pre-post-$(1).flags.$(2) ?= $$(dir.tmp)/pre-js-$(1)-$(2).js: $$(pre-js.js.$(2)) $$(MAKEFILE) @@ -579,6 +645,7 @@ endef # https://github.com/emscripten-core/emscripten/issues/14383 sqlite3.wasm := $(dir.dout)/sqlite3.wasm sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c +sqlite3-wasm.cses := $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) # sqlite3-wasm.o vs sqlite3-wasm.c: building against the latter # (predictably) results in a slightly faster binary. We're close # enough to the target speed requirements that the 500ms makes a @@ -626,18 +693,20 @@ pre-post-common.flags := \ pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) ######################################################################## # SETUP_LIB_BUILD_MODE is a $(call)'able which sets up numerous pieces -# for one of the build modes (vanilla, esm, bundler-friendly). +# for one of the build modes. # -# $1 = build mode name +# $1 = build mode name: one of $(JS_BUILD_MODES) # $2 = 1 for ESM build mode, else 0 # $3 = resulting sqlite-api JS/MJS file # $4 = resulting JS/MJS file # $5 = -D... flags for $(bin.c-pp) -# $6 = emcc -sXYZ flags +# $6 = emcc -sXYZ flags (CURRENTLY UNUSED - was factored out) +# +# emcc.environment.$(1) must be set to a value for the -sENVIRONMENT flag. define SETUP_LIB_BUILD_MODE $(info Setting up build [$(1)]: $(4)) c-pp.D.$(1) := $(5) -pre-js.js.$(1) := $$(dir.api)/pre-js.$(1).js +pre-js.js.$(1) := $$(dir.tmp)/pre-js.$(1).js $$(eval $$(call C-PP.FILTER,$$(pre-js.js.in),$$(pre-js.js.$(1)),$$(c-pp.D.$(1)))) post-js.js.$(1) := $$(dir.tmp)/post-js.$(1).js $$(eval $$(call C-PP.FILTER,$$(post-js.js.in),$$(post-js.js.$(1)),$$(c-pp.D.$(1)))) @@ -652,19 +721,21 @@ pre-post-jses.deps.$(1) := $$(pre-post-jses.deps.common) \ $$(eval $$(call call-make-pre-post,sqlite3,$(1))) emcc.flags.sqlite3.$(1) := $(6) $$(eval $$(call C-PP.FILTER, $$(sqlite3-api.js.in), $(3), $(5))) -$(4): $(3) -$(4): $(3) $$(MAKEFILE) $$(sqlite3-wasm.c) $$(EXPORTED_FUNCTIONS.api) $$(pre-post-sqlite3.deps.$(1)) +$(4): $(3) $$(MAKEFILE) $$(sqlite3-wasm.cses) $$(EXPORTED_FUNCTIONS.api) $$(pre-post-sqlite3.deps.$(1)) @echo "Building $$@ ..." $$(emcc.bin) -o $$@ $$(emcc_opt_full) $$(emcc.flags) \ $$(emcc.jsflags) \ + -sENVIRONMENT=$$(emcc.environment.$(1)) \ $$(pre-post-sqlite3.flags.$(1)) $$(emcc.flags.sqlite3.$(1)) \ - $$(cflags.common) $$(SQLITE_OPT) $$(sqlite3-wasm.c) + $$(cflags.common) $$(SQLITE_OPT) $$(cflags.wasm_extra_init) $$(sqlite3-wasm.cses) @$$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(2)) - @if [ bundler-friendly = $(1) ]; then \ - echo "Patching $(3) for sqlite3.wasm..."; \ - rm -f $$(dir.dout)/sqlite3-bundler-friendly.wasm; \ - sed -i -e 's/sqlite3-bundler-friendly.wasm/sqlite3.wasm/g' $$@ || exit $$$$?; \ - fi + @case $(1) in \ + bundler-friendly|node) \ + echo "Patching $(3) for sqlite3.wasm..."; \ + rm -f $$(dir.dout)/sqlite3-$(1).wasm; \ + sed -i -e 's/sqlite3-$(1).wasm/sqlite3.wasm/g' $$@ || exit $$$$?; \ + ;; \ + esac chmod -x $$(sqlite3.wasm) $$(maybe-wasm-strip) $$(sqlite3.wasm) @ls -la $@ $$(sqlite3.wasm) @@ -680,6 +751,8 @@ sqlite3-api.mjs := $(dir.dout)/sqlite3-api.mjs sqlite3.mjs := $(dir.dout)/sqlite3.mjs sqlite3-api-bundler-friendly.mjs := $(dir.dout)/sqlite3-api-bundler-friendly.mjs sqlite3-bundler-friendly.mjs := $(dir.dout)/sqlite3-bundler-friendly.mjs +sqlite3-api-node.mjs := $(dir.dout)/sqlite3-api-node.mjs +sqlite3-node.mjs := $(dir.dout)/sqlite3-node.mjs # Maintenance reminder: careful not to introduce spaces around args $1, $2 #$(info $(call SETUP_LIB_BUILD_MODE,vanilla,0, $(sqlite3-api.js), $(sqlite3.js))) $(eval $(call SETUP_LIB_BUILD_MODE,vanilla,0, $(sqlite3-api.js), $(sqlite3.js))) @@ -687,7 +760,10 @@ $(eval $(call SETUP_LIB_BUILD_MODE,esm,1, $(sqlite3-api.mjs), $(sqlite3.mjs), \ -Dtarget=es6-module, -sEXPORT_ES6 -sUSE_ES6_IMPORT_META)) $(eval $(call SETUP_LIB_BUILD_MODE,bundler-friendly,1,\ $(sqlite3-api-bundler-friendly.mjs),$(sqlite3-bundler-friendly.mjs),\ - $(c-pp.D.esm) -Dtarget=es6-bundler-friendly, $(emcc.flags.sqlite3.esm))) + $(c-pp.D.esm) -Dtarget=es6-bundler-friendly)) +$(eval $(call SETUP_LIB_BUILD_MODE,node,1,\ + $(sqlite3-api-node.mjs),$(sqlite3-node.mjs),\ + $(c-pp.D.bundler-friendly) -Dtarget=node)) # The various -D... values used by *.c-pp.js include: # # -Dtarget=es6-module: for all ESM module builds @@ -699,18 +775,26 @@ $(eval $(call SETUP_LIB_BUILD_MODE,bundler-friendly,1,\ # as string literals so that bundlers' static-analysis tools can # find those files and include them in their bundles. # +# -Dtarget=es6-module -Dtarget=es6-bundler-friendly -Dtarget=node: is +# intended for use by node.js for node.js, as opposed to by +# node.js on behalf of a browser. Mixing -sENVIRONMENT=web and +# -sENVIRONMENT=node leads to ambiguity and confusion on node's +# part, as it's unable to reliably determine whether the target is +# a browser or node. +# ######################################################################## ######################################################################## -# We have to ensure that we do not build both $(sqlite3*.*js) in -# parallel because both result in the creation of $(sqlite3.wasm). We -# have no way to build just the .mjs file without also building the -# .wasm file because the generated .mjs file has to include info about -# the imports needed by the wasm file, so they have to be built +# We have to ensure that we do not build $(sqlite3*.*js) in parallel +# because they all result in the creation of $(sqlite3.wasm). We have +# no way to build just a .[m]js file without also building the .wasm +# file because the generated .[m]js file has to include info about the +# imports needed by the wasm file, so they have to be built # together. i.e. we're building $(sqlite3.wasm) multiple times, but # that's unavoidable (and harmless, just a waste of build time). $(sqlite3.wasm): $(sqlite3.js) $(sqlite3.mjs): $(sqlite3.js) $(sqlite3-bundler-friendly.mjs): $(sqlite3.mjs) +$(sqlite3-node.mjs): $(sqlite3.mjs) CLEAN_FILES += $(sqlite3.wasm) ######################################################################## @@ -723,7 +807,7 @@ sqlite3-worker1.js.in := $(dir.api)/sqlite3-worker1.c-pp.js sqlite3-worker1-promiser.js.in := $(dir.api)/sqlite3-worker1-promiser.c-pp.js sqlite3-worker1.js := $(dir.dout)/sqlite3-worker1.js sqlite3-worker1-promiser.js := $(dir.dout)/sqlite3-worker1-promiser.js -sqlite3-worker1-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-bundler-friendly.js +sqlite3-worker1-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-bundler-friendly.mjs sqlite3-worker1-promiser-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-promiser-bundler-friendly.js $(eval $(call C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1.js))) $(eval $(call C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1-bundler-friendly.js),\ @@ -772,7 +856,7 @@ emcc.speedtest1 += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY) emcc.speedtest1.common += -sINVOKE_RUN=0 emcc.speedtest1.common += --no-entry emcc.speedtest1.common += -sABORTING_MALLOC -emcc.speedtest1.common += -sSTRICT_JS +emcc.speedtest1.common += -sSTRICT_JS=0 emcc.speedtest1.common += -sMODULARIZE emcc.speedtest1.common += -Wno-limited-postlink-optimizations EXPORTED_FUNCTIONS.speedtest1 := $(abspath $(dir.tmp)/EXPORTED_FUNCTIONS.speedtest1) @@ -803,9 +887,9 @@ speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 # -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app # which runs speedtest1 multiple times. -$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) +$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api.main) @echo "Making $@ ..." - @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api); } > $@ + @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api.main); } > $@ speedtest1.js := $(dir.dout)/speedtest1.js speedtest1.wasm := $(dir.dout)/speedtest1.wasm cflags.speedtest1 := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM @@ -816,9 +900,11 @@ $(speedtest1.js): $(MAKEFILE) $(speedtest1.cses) \ $(EXPORTED_FUNCTIONS.speedtest1) @echo "Building $@ ..." $(emcc.bin) \ - $(emcc.speedtest1) $(emcc.speedtest1.common) \ + $(emcc.speedtest1) -I$(dir $(sqlite3.canonical.c)) \ + $(emcc.speedtest1.common) \ $(cflags.speedtest1) $(pre-post-speedtest1.flags.vanilla) \ $(SQLITE_OPT) \ + -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c) \ $(speedtest1.exit-runtime0) \ -o $@ $(speedtest1.cses) -lm $(maybe-wasm-strip) $(speedtest1.wasm) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see new file mode 100644 index 0000000000..83f3a97dbc --- /dev/null +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see @@ -0,0 +1,5 @@ +_sqlite3_key +_sqlite3_key_v2 +_sqlite3_rekey +_sqlite3_rekey_v2 +_sqlite3_activate_see diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index a577a63e1e..927bf64f9e 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -28,7 +28,7 @@ const toExportForESM = for non-ES6 Module cases but wrong for ES6 modules because those resolve this symbol differently. */ sqlite3InitModule; if(!originalInit){ - throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build."); + throw new Error("Expecting globalThis.sqlite3InitModule to be defined by the Emscripten build."); } /** We need to add some state which our custom Module.locateFile() @@ -41,11 +41,13 @@ const toExportForESM = into the global scope and delete it when sqlite3InitModule() is called. */ - const initModuleState = self.sqlite3InitModuleState = Object.assign(Object.create(null),{ - moduleScript: self?.document?.currentScript, + const initModuleState = globalThis.sqlite3InitModuleState = Object.assign(Object.create(null),{ + moduleScript: globalThis?.document?.currentScript, isWorker: ('undefined' !== typeof WorkerGlobalScope), - location: self.location, - urlParams: new URL(self.location.href).searchParams + location: globalThis.location, + urlParams: globalThis?.location?.href + ? new URL(globalThis.location.href).searchParams + : new URLSearchParams() }); initModuleState.debugModule = initModuleState.urlParams.has('sqlite3.debugModule') @@ -60,14 +62,14 @@ const toExportForESM = initModuleState.sqlite3Dir = li.join('/') + '/'; } - self.sqlite3InitModule = function ff(...args){ - //console.warn("Using replaced sqlite3InitModule()",self.location); + globalThis.sqlite3InitModule = function ff(...args){ + //console.warn("Using replaced sqlite3InitModule()",globalThis.location); return originalInit(...args).then((EmscriptenModule)=>{ - if(self.window!==self && + if('undefined'!==typeof WorkerGlobalScope && (EmscriptenModule['ENVIRONMENT_IS_PTHREAD'] || EmscriptenModule['_pthread_self'] || 'function'===typeof threadAlert - || self.location.pathname.endsWith('.worker.js') + || globalThis?.location?.pathname?.endsWith?.('.worker.js') )){ /** Workaround for wasmfs-generated worker, which calls this routine from each individual thread and requires that its @@ -88,10 +90,10 @@ const toExportForESM = throw e; }); }; - self.sqlite3InitModule.ready = originalInit.ready; + globalThis.sqlite3InitModule.ready = originalInit.ready; - if(self.sqlite3InitModuleState.moduleScript){ - const sim = self.sqlite3InitModuleState; + if(globalThis.sqlite3InitModuleState.moduleScript){ + const sim = globalThis.sqlite3InitModuleState; let src = sim.moduleScript.src.split('/'); src.pop(); sim.scriptDir = src.join('/') + '/'; @@ -99,7 +101,7 @@ const toExportForESM = initModuleState.debugModule('sqlite3InitModuleState =',initModuleState); if(0){ console.warn("Replaced sqlite3InitModule()"); - console.warn("self.location.href =",self.location.href); + console.warn("globalThis.location.href =",globalThis.location.href); if('undefined' !== typeof document){ console.warn("document.currentScript.src =", document?.currentScript?.src); @@ -119,7 +121,7 @@ const toExportForESM = /* AMD modules get injected in a way we cannot override, so we can't handle those here. */ //#endif // !target=es6-module - return self.sqlite3InitModule /* required for ESM */; + return globalThis.sqlite3InitModule /* required for ESM */; })(); //#if target=es6-module export default toExportForESM; diff --git a/ext/wasm/api/pre-js.c-pp.js b/ext/wasm/api/pre-js.c-pp.js index a25c7ce774..878f3e0546 100644 --- a/ext/wasm/api/pre-js.c-pp.js +++ b/ext/wasm/api/pre-js.c-pp.js @@ -6,12 +6,12 @@ */ // See notes in extern-post-js.js -const sqlite3InitModuleState = self.sqlite3InitModuleState +const sqlite3InitModuleState = globalThis.sqlite3InitModuleState || Object.assign(Object.create(null),{ debugModule: ()=>{} }); -delete self.sqlite3InitModuleState; -sqlite3InitModuleState.debugModule('self.location =',self.location); +delete globalThis.sqlite3InitModuleState; +sqlite3InitModuleState.debugModule('globalThis.location =',globalThis.location); //#ifnot target=es6-bundler-friendly /** diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index 7c23f8f894..d38b401bf7 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -25,7 +25,7 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build exports: Module['asm'], memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */ }, - self.sqlite3ApiConfig || {} + globalThis.sqlite3ApiConfig || {} ); /** @@ -33,29 +33,29 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build sqlite3ApiBootstrap(). That decision will be revisited at some point, as we really want client code to be able to call this to configure certain parts. Clients may modify - self.sqlite3ApiBootstrap.defaultConfig to tweak the default + globalThis.sqlite3ApiBootstrap.defaultConfig to tweak the default configuration used by a no-args call to sqlite3ApiBootstrap(), but must have first loaded their WASM module in order to be able to provide the necessary configuration state. */ - //console.warn("self.sqlite3ApiConfig = ",self.sqlite3ApiConfig); - self.sqlite3ApiConfig = SABC; + //console.warn("globalThis.sqlite3ApiConfig = ",globalThis.sqlite3ApiConfig); + globalThis.sqlite3ApiConfig = SABC; let sqlite3; try{ - sqlite3 = self.sqlite3ApiBootstrap(); + sqlite3 = globalThis.sqlite3ApiBootstrap(); }catch(e){ console.error("sqlite3ApiBootstrap() error:",e); throw e; }finally{ - delete self.sqlite3ApiBootstrap; - delete self.sqlite3ApiConfig; + delete globalThis.sqlite3ApiBootstrap; + delete globalThis.sqlite3ApiConfig; } Module.sqlite3 = sqlite3 /* Needed for customized sqlite3InitModule() to be able to pass the sqlite3 object off to the client. */; }else{ console.warn("This is not running in an Emscripten module context, so", - "self.sqlite3ApiBootstrap() is _not_ being called due to lack", + "globalThis.sqlite3ApiBootstrap() is _not_ being called due to lack", "of config info for the WASM environment.", "It must be called manually."); } diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 7db23bacc9..f444ec975c 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -16,13 +16,13 @@ initializes the main API pieces so that the downstream components (e.g. sqlite3-api-oo1.js) have all that they need. */ -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; const toss = (...args)=>{throw new Error(args.join(' '))}; const toss3 = sqlite3.SQLite3Error.toss; const capi = sqlite3.capi, wasm = sqlite3.wasm, util = sqlite3.util; - self.WhWasmUtilInstaller(wasm); - delete self.WhWasmUtilInstaller; + globalThis.WhWasmUtilInstaller(wasm); + delete globalThis.WhWasmUtilInstaller; if(0){ /** @@ -327,6 +327,15 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); } + if(wasm.exports.sqlite3_activate_see instanceof Function){ + wasm.bindingSignatures.push( + ["sqlite3_key", "int", "sqlite3*", "string", "int"], + ["sqlite3_key_v2","int","sqlite3*","string","*","int"], + ["sqlite3_rekey", "int", "sqlite3*", "string", "int"], + ["sqlite3_rekey_v2", "int", "sqlite3*", "string", "*", "int"], + ["sqlite3_activate_see", undefined, "string"] + ); + } /** Functions which require BigInt (int64) support are separated from the others because we need to conditionally bind them or apply @@ -605,7 +614,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Install JS<->C struct bindings for the non-opaque struct types we need... */ - sqlite3.StructBinder = self.Jaccwabyt({ + sqlite3.StructBinder = globalThis.Jaccwabyt({ heap: 0 ? wasm.memory : wasm.heap8u, alloc: wasm.alloc, dealloc: wasm.dealloc, @@ -613,7 +622,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ memberPrefix: /* Never change this: this prefix is baked into any amount of code and client-facing docs. */ '$' }); - delete self.Jaccwabyt; + delete globalThis.Jaccwabyt; {// wasm.xWrap() bindings... diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 914497602e..ac6678c88e 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -12,9 +12,9 @@ This file contains the so-called OO #1 API wrapper for the sqlite3 WASM build. It requires that sqlite3-api-glue.js has already run - and it installs its deliverable as self.sqlite3.oo1. + and it installs its deliverable as globalThis.sqlite3.oo1. */ -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = (...args)=>{throw new Error(args.join(' '))}; const toss3 = (...args)=>{throw new sqlite3.SQLite3Error(...args)}; diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index b08ad7a7ce..c882d5b247 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -29,7 +29,7 @@ exposed by this API. It is intended to be called one time at the end of the API amalgamation process, passed configuration details for the current environment, and then optionally be removed from - the global object using `delete self.sqlite3ApiBootstrap`. + the global object using `delete globalThis.sqlite3ApiBootstrap`. This function is not intended for client-level use. It is intended for use in creating bundles configured for specific WASM @@ -58,7 +58,7 @@ WASM-exported memory. - `bigIntEnabled`: true if BigInt support is enabled. Defaults to - true if `self.BigInt64Array` is available, else false. Some APIs + true if `globalThis.BigInt64Array` is available, else false. Some APIs will throw exceptions if called without BigInt support, as BigInt is required for marshalling C-side int64 into and out of JS. (Sidebar: it is technically possible to add int64 support via @@ -100,8 +100,8 @@ */ 'use strict'; -self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( - apiConfig = (self.sqlite3ApiConfig || sqlite3ApiBootstrap.defaultConfig) +globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( + apiConfig = (globalThis.sqlite3ApiConfig || sqlite3ApiBootstrap.defaultConfig) ){ if(sqlite3ApiBootstrap.sqlite3){ /* already initalized */ console.warn("sqlite3ApiBootstrap() called multiple times.", @@ -117,7 +117,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( -sWASM_BIGINT=1, else it will not. */ return !!Module.HEAPU64; } - return !!self.BigInt64Array; + return !!globalThis.BigInt64Array; })(), debug: console.debug.bind(console), warn: console.warn.bind(console), @@ -772,7 +772,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( isBindableTypedArray, isInt32, isSQLableTypedArray, isTypedArray, typedArrayToString, - isUIThread: ()=>(self.window===self && !!self.document), + isUIThread: ()=>(globalThis.window===globalThis && !!globalThis.document), // is this true for ESM?: 'undefined'===typeof WorkerGlobalScope isSharedTypedArray, toss: function(...args){throw new Error(args.join(' '))}, @@ -1203,9 +1203,9 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( console.error("sqlite3_wasmfs_opfs_dir() can no longer work due "+ "to incompatible WASMFS changes. It will be removed."); if(!pdir - || !self.FileSystemHandle - || !self.FileSystemDirectoryHandle - || !self.FileSystemFileHandle){ + || !globalThis.FileSystemHandle + || !globalThis.FileSystemDirectoryHandle + || !globalThis.FileSystemFileHandle){ return __wasmfsOpfsDir = ""; } try{ @@ -1461,8 +1461,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( const rc = Object.create(null); rc.prefix = 'kvvfs-'+which; rc.stores = []; - if('session'===which || ""===which) rc.stores.push(self.sessionStorage); - if('local'===which || ""===which) rc.stores.push(self.localStorage); + if('session'===which || ""===which) rc.stores.push(globalThis.sessionStorage); + if('local'===which || ""===which) rc.stores.push(globalThis.localStorage); return rc; }; @@ -1537,8 +1537,11 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( Full docs: https://sqlite.org/c3ref/db_config.html Returns capi.SQLITE_MISUSE if op is not a valid operation ID. + + The variants which take `(int, int*)` arguments treat a + missing or falsy pointer argument as 0. */ - capi.sqlite3_db_config = function f(pDb, op, ...args){ + capi.sqlite3_db_config = function(pDb, op, ...args){ if(!this.s){ this.s = wasm.xWrap('sqlite3_wasm_db_config_s','int', ['sqlite3*', 'int', 'string:static'] @@ -1548,31 +1551,32 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( this.ip = wasm.xWrap('sqlite3_wasm_db_config_ip','int', ['sqlite3*', 'int', 'int','*']); } - const c = capi; switch(op){ - case c.SQLITE_DBCONFIG_ENABLE_FKEY: - case c.SQLITE_DBCONFIG_ENABLE_TRIGGER: - case c.SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: - case c.SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: - case c.SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: - case c.SQLITE_DBCONFIG_ENABLE_QPSG: - case c.SQLITE_DBCONFIG_TRIGGER_EQP: - case c.SQLITE_DBCONFIG_RESET_DATABASE: - case c.SQLITE_DBCONFIG_DEFENSIVE: - case c.SQLITE_DBCONFIG_WRITABLE_SCHEMA: - case c.SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: - case c.SQLITE_DBCONFIG_DQS_DML: - case c.SQLITE_DBCONFIG_DQS_DDL: - case c.SQLITE_DBCONFIG_ENABLE_VIEW: - case c.SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: - case c.SQLITE_DBCONFIG_TRUSTED_SCHEMA: + case capi.SQLITE_DBCONFIG_ENABLE_FKEY: + case capi.SQLITE_DBCONFIG_ENABLE_TRIGGER: + case capi.SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: + case capi.SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: + case capi.SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: + case capi.SQLITE_DBCONFIG_ENABLE_QPSG: + case capi.SQLITE_DBCONFIG_TRIGGER_EQP: + case capi.SQLITE_DBCONFIG_RESET_DATABASE: + case capi.SQLITE_DBCONFIG_DEFENSIVE: + case capi.SQLITE_DBCONFIG_WRITABLE_SCHEMA: + case capi.SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: + case capi.SQLITE_DBCONFIG_DQS_DML: + case capi.SQLITE_DBCONFIG_DQS_DDL: + case capi.SQLITE_DBCONFIG_ENABLE_VIEW: + case capi.SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: + case capi.SQLITE_DBCONFIG_TRUSTED_SCHEMA: + case capi.SQLITE_DBCONFIG_STMT_SCANSTATUS: + case capi.SQLITE_DBCONFIG_REVERSE_SCANORDER: return this.ip(pDb, op, args[0], args[1] || 0); - case c.SQLITE_DBCONFIG_LOOKASIDE: + case capi.SQLITE_DBCONFIG_LOOKASIDE: return this.pii(pDb, op, args[0], args[1], args[2]); - case c.SQLITE_DBCONFIG_MAINDBNAME: + case capi.SQLITE_DBCONFIG_MAINDBNAME: return this.s(pDb, op, args[0]); default: - return c.SQLITE_MISUSE; + return capi.SQLITE_MISUSE; } }.bind(Object.create(null)); @@ -1962,7 +1966,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( return sqlite3; }/*sqlite3ApiBootstrap()*/; /** - self.sqlite3ApiBootstrap.initializers is an internal detail used by + globalThis.sqlite3ApiBootstrap.initializers is an internal detail used by the various pieces of the sqlite3 API's amalgamation process. It must not be modified by client code except when plugging such code into the amalgamation process. @@ -1980,14 +1984,14 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( utilized until the whwasmutil.js part is plugged in via sqlite3-api-glue.js. */ -self.sqlite3ApiBootstrap.initializers = []; +globalThis.sqlite3ApiBootstrap.initializers = []; /** - self.sqlite3ApiBootstrap.initializersAsync is an internal detail + globalThis.sqlite3ApiBootstrap.initializersAsync is an internal detail used by the sqlite3 API's amalgamation process. It must not be modified by client code except when plugging such code into the amalgamation process. - The counterpart of self.sqlite3ApiBootstrap.initializers, + The counterpart of globalThis.sqlite3ApiBootstrap.initializers, specifically for initializers which are asynchronous. All entries in this list must be either async functions, non-async functions which return a Promise, or a Promise. Each function in the list is called @@ -1999,10 +2003,10 @@ self.sqlite3ApiBootstrap.initializers = []; This list is not processed until the client calls sqlite3.asyncPostInit(). This means, for example, that intializers - added to self.sqlite3ApiBootstrap.initializers may push entries to + added to globalThis.sqlite3ApiBootstrap.initializers may push entries to this list. */ -self.sqlite3ApiBootstrap.initializersAsync = []; +globalThis.sqlite3ApiBootstrap.initializersAsync = []; /** Client code may assign sqlite3ApiBootstrap.defaultConfig an object-type value before calling sqlite3ApiBootstrap() (without @@ -2012,13 +2016,12 @@ self.sqlite3ApiBootstrap.initializersAsync = []; an environment-suitable configuration without having to define a new global-scope symbol. */ -self.sqlite3ApiBootstrap.defaultConfig = Object.create(null); +globalThis.sqlite3ApiBootstrap.defaultConfig = Object.create(null); /** Placeholder: gets installed by the first call to - self.sqlite3ApiBootstrap(). However, it is recommended that the + globalThis.sqlite3ApiBootstrap(). However, it is recommended that the caller of sqlite3ApiBootstrap() capture its return value and delete - self.sqlite3ApiBootstrap after calling it. It returns the same + globalThis.sqlite3ApiBootstrap after calling it. It returns the same value which will be stored here. */ -self.sqlite3ApiBootstrap.sqlite3 = undefined; - +globalThis.sqlite3ApiBootstrap.sqlite3 = undefined; diff --git a/ext/wasm/api/sqlite3-api-worker1.js b/ext/wasm/api/sqlite3-api-worker1.js index f82be6cd09..d1c63c96ee 100644 --- a/ext/wasm/api/sqlite3-api-worker1.js +++ b/ext/wasm/api/sqlite3-api-worker1.js @@ -313,11 +313,11 @@ options.columnNames may be populated by the call to db.exec(). */ -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3.initWorker1API = function(){ 'use strict'; const toss = (...args)=>{throw new Error(args.join(' '))}; - if('function' !== typeof importScripts){ + if(!(globalThis.WorkerGlobalScope instanceof Function)){ toss("initWorker1API() must be run from a Worker thread."); } const self = this.self; @@ -382,10 +382,10 @@ sqlite3.initWorker1API = function(){ */ post: function(msg,xferList){ if(xferList && xferList.length){ - self.postMessage( msg, Array.from(xferList) ); + globalThis.postMessage( msg, Array.from(xferList) ); xferList.length = 0; }else{ - self.postMessage(msg); + globalThis.postMessage(msg); } }, /** Map of DB IDs to DBs. */ @@ -589,7 +589,7 @@ sqlite3.initWorker1API = function(){ } }/*wMsgHandler*/; - self.onmessage = async function(ev){ + globalThis.onmessage = async function(ev){ ev = ev.data; let result, dbId = ev.dbId, evType = ev.type; const arrivalTime = performance.now(); @@ -637,6 +637,6 @@ sqlite3.initWorker1API = function(){ result: result }, wState.xfer); }; - self.postMessage({type:'sqlite3-api',result:'worker1-ready'}); + globalThis.postMessage({type:'sqlite3-api',result:'worker1-ready'}); }.bind({self, sqlite3}); }); diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 1456ae08d2..ddcad8f61c 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -50,10 +50,10 @@ const wPost = (type,...args)=>postMessage({type, payload:args}); const installAsyncProxy = function(self){ const toss = function(...args){throw new Error(args.join(' '))}; - if(self.window === self){ + if(globalThis.window === globalThis){ toss("This code cannot run from the main thread.", "Load it as a Worker from a separate Worker."); - }else if(!navigator.storage.getDirectory){ + }else if(!navigator?.storage?.getDirectory){ toss("This API requires navigator.storage.getDirectory."); } @@ -106,8 +106,8 @@ const installAsyncProxy = function(self){ w += m.wait; m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; } - console.log(self.location.href, - "metrics for",self.location.href,":\n", + console.log(globalThis?.location?.href, + "metrics for",globalThis?.location?.href,":\n", metrics, "\nTotal of",n,"op(s) for",t,"ms", "approx",w,"ms spent waiting on OPFS APIs."); @@ -843,7 +843,7 @@ const installAsyncProxy = function(self){ navigator.storage.getDirectory().then(function(d){ state.rootDir = d; - self.onmessage = function({data}){ + globalThis.onmessage = function({data}){ switch(data.type){ case 'opfs-async-init':{ /* Receive shared state from synchronous partner */ @@ -880,17 +880,17 @@ const installAsyncProxy = function(self){ wPost('opfs-async-loaded'); }).catch((e)=>error("error initializing OPFS asyncer:",e)); }/*installAsyncProxy()*/; -if(!self.SharedArrayBuffer){ +if(!globalThis.SharedArrayBuffer){ wPost('opfs-unavailable', "Missing SharedArrayBuffer API.", "The server must emit the COOP/COEP response headers to enable that."); -}else if(!self.Atomics){ +}else if(!globalThis.Atomics){ wPost('opfs-unavailable', "Missing Atomics API.", "The server must emit the COOP/COEP response headers to enable that."); -}else if(!self.FileSystemHandle || - !self.FileSystemDirectoryHandle || - !self.FileSystemFileHandle || - !self.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator.storage.getDirectory){ +}else if(!globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle || + !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator?.storage?.getDirectory){ wPost('opfs-unavailable',"Missing required OPFS APIs."); }else{ installAsyncProxy(self); diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js index 10be8ebce4..80ab7c5b04 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -15,10 +15,13 @@ with its virtual table counterpart, sqlite3.vtab. */ 'use strict'; -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss3; const vfs = Object.create(null), vtab = Object.create(null); + const StructBinder = sqlite3.StructBinder + /* we require a local alias b/c StructBinder is removed from the sqlite3 + object during the final steps of the API cleanup. */; sqlite3.vfs = vfs; sqlite3.vtab = vtab; @@ -112,7 +115,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const installMethod = function callee( tgt, name, func, applyArgcCheck = callee.installMethodArgcCheck ){ - if(!(tgt instanceof sqlite3.StructBinder.StructType)){ + if(!(tgt instanceof StructBinder.StructType)){ toss("Usage error: target object is-not-a StructType."); }else if(!(func instanceof Function) && !wasm.isPtr(func)){ toss("Usage errror: expecting a Function or WASM pointer to one."); @@ -132,7 +135,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; /* An ondispose() callback for use with - sqlite3.StructBinder-created types. */ + StructBinder-created types. */ callee.removeFuncList = function(){ if(this.ondispose.__removeFuncList){ this.ondispose.__removeFuncList.forEach( @@ -221,7 +224,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ and the first is an object, it's instead equivalent to calling installMethods(this,...arguments). */ - sqlite3.StructBinder.StructType.prototype.installMethod = function callee( + StructBinder.StructType.prototype.installMethod = function callee( name, func, applyArgcCheck = installMethod.installMethodArgcCheck ){ return (arguments.length < 3 && name && 'object'===typeof name) @@ -233,7 +236,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Equivalent to calling installMethods() with a first argument of this object. */ - sqlite3.StructBinder.StructType.prototype.installMethods = function( + StructBinder.StructType.prototype.installMethods = function( methods, applyArgcCheck = installMethod.installMethodArgcCheck ){ return installMethods(this, methods, applyArgcCheck); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 3e3255b0c8..5c584702d8 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -18,7 +18,7 @@ after sqlite3-api-oo1.js and before sqlite3-api-cleanup.js. */ 'use strict'; -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** installOpfsVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs @@ -76,23 +76,23 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ `opfs` property, containing several OPFS-specific utilities. */ const installOpfsVfs = function callee(options){ - if(!self.SharedArrayBuffer - || !self.Atomics){ + if(!globalThis.SharedArrayBuffer + || !globalThis.Atomics){ return Promise.reject( new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ "The server must emit the COOP/COEP response headers to enable those. "+ "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") ); - }else if(self.window===self && self.document){ + }else if('undefined'===typeof WorkerGlobalScope){ return Promise.reject( new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ "because it requires Atomics.wait().") ); - }else if(!self.FileSystemHandle || - !self.FileSystemDirectoryHandle || - !self.FileSystemFileHandle || - !self.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator.storage.getDirectory){ + }else if(!globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle || + !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator?.storage?.getDirectory){ return Promise.reject( new Error("Missing required OPFS APIs.") ); @@ -100,7 +100,7 @@ const installOpfsVfs = function callee(options){ if(!options || 'object'!==typeof options){ options = Object.create(null); } - const urlParams = new URL(self.location.href).searchParams; + const urlParams = new URL(globalThis.location.href).searchParams; if(undefined===options.verbose){ options.verbose = urlParams.has('opfs-verbose') ? (+urlParams.get('opfs-verbose') || 2) : 1; @@ -112,16 +112,16 @@ const installOpfsVfs = function callee(options){ options.proxyUri = callee.defaultProxyUri; } - //sqlite3.config.warn("OPFS options =",options,self.location); + //sqlite3.config.warn("OPFS options =",options,globalThis.location); if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } - const thePromise = new Promise(function(promiseResolve, promiseReject_){ + const thePromise = new Promise(function(promiseResolve_, promiseReject_){ const loggers = { - 0:sqlite3.config.error.bind(console), - 1:sqlite3.config.warn.bind(console), - 2:sqlite3.config.log.bind(console) + 0:sqlite3.config.error, + 1:sqlite3.config.warn, + 2:sqlite3.config.log }; const logImpl = (level,...args)=>{ if(options.verbose>level) loggers[level]("OPFS syncer:",...args); @@ -149,11 +149,11 @@ const installOpfsVfs = function callee(options){ Returns true if _this_ thread has access to the OPFS APIs. */ const thisThreadHasOPFS = ()=>{ - return self.FileSystemHandle && - self.FileSystemDirectoryHandle && - self.FileSystemFileHandle && - self.FileSystemFileHandle.prototype.createSyncAccessHandle && - navigator.storage.getDirectory; + return globalThis.FileSystemHandle && + globalThis.FileSystemDirectoryHandle && + globalThis.FileSystemFileHandle && + globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator?.storage?.getDirectory; }; /** @@ -171,8 +171,8 @@ const installOpfsVfs = function callee(options){ m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; } - sqlite3.config.log(self.location.href, - "metrics for",self.location.href,":",metrics, + sqlite3.config.log(globalThis.location.href, + "metrics for",globalThis.location.href,":",metrics, "\nTotal of",n,"op(s) for",t, "ms (incl. "+w+" ms of waiting on the async side)"); sqlite3.config.log("Serialization metrics:",metrics.s11n); @@ -193,10 +193,16 @@ const installOpfsVfs = function callee(options){ }/*metrics*/; const opfsVfs = new sqlite3_vfs(); const opfsIoMethods = new sqlite3_io_methods(); - const promiseReject = function(err){ + let promiseWasRejected = undefined; + const promiseReject = (err)=>{ + promiseWasRejected = true; opfsVfs.dispose(); return promiseReject_(err); }; + const promiseResolve = (value)=>{ + promiseWasRejected = false; + return promiseResolve_(value); + }; const W = //#if target=es6-bundler-friendly new Worker(new URL("sqlite3-opfs-async-proxy.js", import.meta.url)); @@ -205,6 +211,18 @@ const installOpfsVfs = function callee(options){ //#else new Worker(options.proxyUri); //#endif + setTimeout(()=>{ + /* At attempt to work around a browser-specific quirk in which + the Worker load is failing in such a way that we neither + resolve nor reject it. This workaround gives that resolve/reject + a time limit and rejects if that timer expires. Discussion: + https://sqlite.org/forum/forumpost/a708c98dcb3ef */ + if(undefined===promiseWasRejected){ + promiseReject( + new Error("Timeout while waiting for OPFS async proxy worker.") + ); + } + }, 4000); W._originalOnError = W.onerror /* will be restored later */; W.onerror = function(err){ // The error object doesn't contain any useful info when the @@ -335,7 +353,6 @@ const installOpfsVfs = function callee(options){ state.opIds.xClose = i++; state.opIds.xDelete = i++; state.opIds.xDeleteNoWait = i++; - state.opIds.xFileControl = i++; state.opIds.xFileSize = i++; state.opIds.xLock = i++; state.opIds.xOpen = i++; @@ -700,12 +717,9 @@ const installOpfsVfs = function callee(options){ return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; }, xFileControl: function(pFile, opId, pArg){ - mTimeStart('xFileControl'); - const rc = (capi.SQLITE_FCNTL_SYNC===opId) - ? opRun('xSync', pFile, 0) - : capi.SQLITE_NOTFOUND; - mTimeEnd(); - return rc; + /*mTimeStart('xFileControl'); + mTimeEnd();*/ + return capi.SQLITE_NOTFOUND; }, xFileSize: function(pFile,pSz64){ mTimeStart('xFileSize'); @@ -761,8 +775,11 @@ const installOpfsVfs = function callee(options){ return rc; }, xSync: function(pFile,flags){ + mTimeStart('xSync'); ++metrics.xSync.count; - return 0; // impl'd in xFileControl() + const rc = opRun('xSync', pFile, flags); + mTimeEnd(); + return rc; }, xTruncate: function(pFile,sz64){ mTimeStart('xTruncate'); @@ -1171,7 +1188,15 @@ const installOpfsVfs = function callee(options){ /* Truncate journal mode is faster than delete for this vfs, per speedtest1. That gap seems to have closed with Chrome version 108 or 109, but "persist" is very roughly 5-6% - faster than truncate in initial tests. */ + faster than truncate in initial tests. + + For later analysis: Roy Hashimoto notes that TRUNCATE + and PERSIST modes may decrease OPFS concurrency because + multiple connections can open the journal file in those + modes: + + https://github.com/rhashimoto/wa-sqlite/issues/68 + */ "pragma journal_mode=persist;", /* This vfs benefits hugely from cache on moderate/large @@ -1261,14 +1286,18 @@ const installOpfsVfs = function callee(options){ promiseReject(new Error(data.payload.join(' '))); break; case 'opfs-async-loaded': - /*Arrives as soon as the asyc proxy finishes loading. - Pass our config and shared state on to the async worker.*/ + /* Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async + worker. */ W.postMessage({type: 'opfs-async-init',args: state}); break; - case 'opfs-async-inited':{ - /*Indicates that the async partner has received the 'init' - and has finished initializing, so the real work can - begin...*/ + case 'opfs-async-inited': { + /* Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin... */ + if(true===promiseWasRejected){ + break /* promise was already rejected via timer */; + } try { sqlite3.vfs.installVfs({ io: {struct: opfsIoMethods, methods: ioSyncWrappers}, @@ -1300,10 +1329,15 @@ const installOpfsVfs = function callee(options){ } break; } - default: - promiseReject(e); - error("Unexpected message from the async worker:",data); + default: { + const errMsg = ( + "Unexpected message from the OPFS async worker: " + + JSON.stringify(data) + ); + error(errMsg); + promiseReject(new Error(errMsg)); break; + } }/*switch(data.type)*/ }/*W.onmessage()*/; })/*thePromise*/; @@ -1311,7 +1345,7 @@ const installOpfsVfs = function callee(options){ }/*installOpfsVfs()*/; installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; -self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ +globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ try{ let proxyJs = installOpfsVfs.defaultProxyUri; if(sqlite3.scriptInfo.sqlite3Dir){ diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index e7513509ca..dbe594dc3d 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -163,15 +163,14 @@ # define SQLITE_USE_URI 1 #endif -#include -#include "sqlite3.c" /* yes, .c instead of .h. */ - -#if defined(__EMSCRIPTEN__) -# include +#ifdef SQLITE_WASM_EXTRA_INIT +# define SQLITE_EXTRA_INIT sqlite3_wasm_extra_init #endif +#include + /* -** SQLITE_WASM_KEEP is functionally identical to EMSCRIPTEN_KEEPALIVE +** SQLITE_WASM_EXPORT is functionally identical to EMSCRIPTEN_KEEPALIVE ** but is not Emscripten-specific. It explicitly marks functions for ** export into the target wasm file without requiring explicit listing ** of those functions in Emscripten's -sEXPORTED_FUNCTIONS=... list @@ -193,10 +192,34 @@ ** this writing we are tied to Emscripten for various reasons ** and cannot test the library with other build environments. */ -#define SQLITE_WASM_KEEP __attribute__((used,visibility("default"))) +#define SQLITE_WASM_EXPORT __attribute__((used,visibility("default"))) // See also: //__attribute__((export_name("theExportedName"), used, visibility("default"))) +/* +** Which sqlite3.c we're using needs to be configurable to enable +** building against a custom copy, e.g. the SEE variant. Note that we +** #include the .c file, rather than the header, so that the WASM +** extensions have access to private API internals. +** +** The caveat here is that custom variants need to account for +** exporting any necessary symbols (e.g. sqlite3_activate_see()). We +** cannot export them from here using SQLITE_WASM_EXPORT because that +** attribute (apparently) has to be part of the function definition. +*/ +#ifndef SQLITE_C +# define SQLITE_C sqlite3.c /* yes, .c instead of .h. */ +#endif +#define INC__STRINGIFY_(f) #f +#define INC__STRINGIFY(f) INC__STRINGIFY_(f) +#include INC__STRINGIFY(SQLITE_C) +#undef INC__STRINGIFY_ +#undef INC__STRINGIFY +#undef SQLITE_C + +#if defined(__EMSCRIPTEN__) +# include +#endif #if 0 /* @@ -210,24 +233,24 @@ ** Another option is to malloc() a chunk of our own and call that our ** "stack". */ -SQLITE_WASM_KEEP void * sqlite3_wasm_stack_end(void){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_end(void){ extern void __heap_base /* see https://stackoverflow.com/questions/10038964 */; return &__heap_base; } -SQLITE_WASM_KEEP void * sqlite3_wasm_stack_begin(void){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_begin(void){ extern void __data_end; return &__data_end; } static void * pWasmStackPtr = 0; -SQLITE_WASM_KEEP void * sqlite3_wasm_stack_ptr(void){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_ptr(void){ if(!pWasmStackPtr) pWasmStackPtr = sqlite3_wasm_stack_end(); return pWasmStackPtr; } -SQLITE_WASM_KEEP void sqlite3_wasm_stack_restore(void * p){ +SQLITE_WASM_EXPORT void sqlite3_wasm_stack_restore(void * p){ pWasmStackPtr = p; } -SQLITE_WASM_KEEP void * sqlite3_wasm_stack_alloc(int n){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_alloc(int n){ if(n<=0) return 0; n = (n + 7) & ~7 /* align to 8-byte boundary */; unsigned char * const p = (unsigned char *)sqlite3_wasm_stack_ptr(); @@ -263,14 +286,14 @@ static struct { /* ** Returns the current pstack position. */ -SQLITE_WASM_KEEP void * sqlite3_wasm_pstack_ptr(void){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_pstack_ptr(void){ return PStack.pPos; } /* ** Sets the pstack position poitner to p. Results are undefined if the ** given value did not come from sqlite3_wasm_pstack_ptr(). */ -SQLITE_WASM_KEEP void sqlite3_wasm_pstack_restore(unsigned char * p){ +SQLITE_WASM_EXPORT void sqlite3_wasm_pstack_restore(unsigned char * p){ assert(p>=PStack.pBegin && p<=PStack.pEnd && p>=PStack.pPos); assert(0==(p & 0x7)); if(p>=PStack.pBegin && p<=PStack.pEnd /*&& p>=PStack.pPos*/){ @@ -285,7 +308,7 @@ SQLITE_WASM_KEEP void sqlite3_wasm_pstack_restore(unsigned char * p){ ** JS code from having to do so, and most uses of the pstack will ** call for doing so). */ -SQLITE_WASM_KEEP void * sqlite3_wasm_pstack_alloc(int n){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_pstack_alloc(int n){ if( n<=0 ) return 0; //if( n & 0x7 ) n += 8 - (n & 0x7) /* align to 8-byte boundary */; n = (n + 7) & ~7 /* align to 8-byte boundary */; @@ -298,7 +321,7 @@ SQLITE_WASM_KEEP void * sqlite3_wasm_pstack_alloc(int n){ ** Return the number of bytes left which can be ** sqlite3_wasm_pstack_alloc()'d. */ -SQLITE_WASM_KEEP int sqlite3_wasm_pstack_remaining(void){ +SQLITE_WASM_EXPORT int sqlite3_wasm_pstack_remaining(void){ assert(PStack.pPos >= PStack.pBegin); assert(PStack.pPos <= PStack.pEnd); return (int)(PStack.pPos - PStack.pBegin); @@ -309,7 +332,7 @@ SQLITE_WASM_KEEP int sqlite3_wasm_pstack_remaining(void){ ** any space which is currently allocated. This value is a ** compile-time constant. */ -SQLITE_WASM_KEEP int sqlite3_wasm_pstack_quota(void){ +SQLITE_WASM_EXPORT int sqlite3_wasm_pstack_quota(void){ return (int)(PStack.pEnd - PStack.pBegin); } @@ -327,7 +350,7 @@ SQLITE_WASM_KEEP int sqlite3_wasm_pstack_quota(void){ ** ** Returns err_code. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_error(sqlite3*db, int err_code, const char *zMsg){ if( db!=0 ){ if( 0!=zMsg ){ @@ -349,7 +372,7 @@ struct WasmTestStruct { void (*xFunc)(void*); }; typedef struct WasmTestStruct WasmTestStruct; -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT void sqlite3_wasm_test_struct(WasmTestStruct * s){ if(s){ s->v4 *= 2; @@ -377,7 +400,7 @@ void sqlite3_wasm_test_struct(WasmTestStruct * s){ ** buffer is not large enough for the generated JSON and needs to be ** increased. In debug builds that will trigger an assert(). */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT const char * sqlite3_wasm_enum_json(void){ static char aBuffer[1024 * 20] = {0} /* where the JSON goes */; int n = 0, nChildren = 0, nStruct = 0 @@ -472,6 +495,7 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_CHANGESETSTART_INVERT); DefInt(SQLITE_CHANGESETAPPLY_NOSAVEPOINT); DefInt(SQLITE_CHANGESETAPPLY_INVERT); + DefInt(SQLITE_CHANGESETAPPLY_IGNORENOOP); DefInt(SQLITE_CHANGESET_DATA); DefInt(SQLITE_CHANGESET_NOTFOUND); @@ -543,6 +567,8 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_DBCONFIG_ENABLE_VIEW); DefInt(SQLITE_DBCONFIG_LEGACY_FILE_FORMAT); DefInt(SQLITE_DBCONFIG_TRUSTED_SCHEMA); + DefInt(SQLITE_DBCONFIG_STMT_SCANSTATUS); + DefInt(SQLITE_DBCONFIG_REVERSE_SCANORDER); DefInt(SQLITE_DBCONFIG_MAX); } _DefGroup; @@ -614,6 +640,7 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_FCNTL_CKPT_START); DefInt(SQLITE_FCNTL_EXTERNAL_READER); DefInt(SQLITE_FCNTL_CKSM_FILE); + DefInt(SQLITE_FCNTL_RESET_CACHE); } _DefGroup; DefGroup(flock) { @@ -903,6 +930,7 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_VTAB_CONSTRAINT_SUPPORT); DefInt(SQLITE_VTAB_INNOCUOUS); DefInt(SQLITE_VTAB_DIRECTONLY); + DefInt(SQLITE_VTAB_USES_ALL_SCHEMAS); DefInt(SQLITE_ROLLBACK); //DefInt(SQLITE_IGNORE); // Also used by sqlite3_authorizer() callback DefInt(SQLITE_FAIL); @@ -1182,7 +1210,7 @@ const char * sqlite3_wasm_enum_json(void){ ** method, SQLITE_MISUSE is returned, else the result of the xDelete() ** call is returned. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ int rc = SQLITE_MISUSE /* ??? */; if( 0==pVfs && 0!=zName ) pVfs = sqlite3_vfs_find(0); @@ -1200,7 +1228,7 @@ int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ ** defaulting to "main" if zDbName is 0. Returns 0 if no db with the ** given name is open. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT sqlite3_vfs * sqlite3_wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ sqlite3_vfs * pVfs = 0; sqlite3_file_control(pDb, zDbName ? zDbName : "main", @@ -1223,7 +1251,7 @@ sqlite3_vfs * sqlite3_wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ ** Returns 0 on success, an SQLITE_xxx code on error. Returns ** SQLITE_MISUSE if pDb is NULL. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_reset(sqlite3 *pDb){ int rc = SQLITE_MISUSE; if( pDb ){ @@ -1254,7 +1282,7 @@ int sqlite3_wasm_db_reset(sqlite3 *pDb){ ** sqlite3_wasm_db_serialize() is arguably the better way to achieve ** this. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_export_chunked( sqlite3* pDb, int (*xCallback)(unsigned const char *zOut, int n) ){ sqlite3_int64 nSize = 0; @@ -1305,7 +1333,7 @@ int sqlite3_wasm_db_export_chunked( sqlite3* pDb, ** If `*pOut` is not NULL, the caller is responsible for passing it to ** sqlite3_free() to free it. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, unsigned char **pOut, sqlite3_int64 *nOut, unsigned int mFlags ){ @@ -1362,7 +1390,7 @@ int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, ** portability, so that the API can still work in builds where BigInt ** support is disabled or unavailable. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, const char *zFilename, const unsigned char * pData, @@ -1446,7 +1474,7 @@ int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, ** NUL-terminated pointer to that string. It is up to the caller to ** use sqlite3_wasm_pstack_restore() to free the returned pointer. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT char * sqlite3_wasm_kvvfsMakeKeyOnPstack(const char *zClass, const char *zKeyIn){ assert(sqlite3KvvfsMethods.nKeySize>24); @@ -1465,7 +1493,7 @@ char * sqlite3_wasm_kvvfsMakeKeyOnPstack(const char *zClass, ** Returns the pointer to the singleton object which holds the kvvfs ** I/O methods and associated state. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT sqlite3_kvvfs_methods * sqlite3_wasm_kvvfs_methods(void){ return &sqlite3KvvfsMethods; } @@ -1480,7 +1508,7 @@ sqlite3_kvvfs_methods * sqlite3_wasm_kvvfs_methods(void){ ** sqlite3_vtab_config(), or SQLITE_MISUSE if the 2nd arg is not a ** valid value. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_vtab_config(sqlite3 *pDb, int op, int arg){ switch(op){ case SQLITE_VTAB_DIRECTONLY: @@ -1500,7 +1528,7 @@ int sqlite3_wasm_vtab_config(sqlite3 *pDb, int op, int arg){ ** Wrapper for the variants of sqlite3_db_config() which take ** (int,int*) variadic args. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ switch(op){ case SQLITE_DBCONFIG_ENABLE_FKEY: @@ -1519,6 +1547,8 @@ int sqlite3_wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ case SQLITE_DBCONFIG_ENABLE_VIEW: case SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: case SQLITE_DBCONFIG_TRUSTED_SCHEMA: + case SQLITE_DBCONFIG_STMT_SCANSTATUS: + case SQLITE_DBCONFIG_REVERSE_SCANORDER: return sqlite3_db_config(pDb, op, arg1, pArg2); default: return SQLITE_MISUSE; } @@ -1531,7 +1561,7 @@ int sqlite3_wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ ** Wrapper for the variants of sqlite3_db_config() which take ** (void*,int,int) variadic args. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int arg3){ switch(op){ case SQLITE_DBCONFIG_LOOKASIDE: @@ -1547,7 +1577,7 @@ int sqlite3_wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int ** Wrapper for the variants of sqlite3_db_config() which take ** (const char *) variadic args. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ switch(op){ case SQLITE_DBCONFIG_MAINDBNAME: @@ -1564,7 +1594,7 @@ int sqlite3_wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ ** Binding for combinations of sqlite3_config() arguments which take ** a single integer argument. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_config_i(int op, int arg){ return sqlite3_config(op, arg); } @@ -1576,7 +1606,7 @@ int sqlite3_wasm_config_i(int op, int arg){ ** Binding for combinations of sqlite3_config() arguments which take ** two int arguments. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_config_ii(int op, int arg1, int arg2){ return sqlite3_config(op, arg1, arg2); } @@ -1588,7 +1618,7 @@ int sqlite3_wasm_config_ii(int op, int arg1, int arg2){ ** Binding for combinations of sqlite3_config() arguments which take ** a single i64 argument. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_config_j(int op, sqlite3_int64 arg){ return sqlite3_config(op, arg); } @@ -1617,7 +1647,7 @@ int sqlite3_wasm_config_j(int op, sqlite3_int64 arg){ ** Safari-specific quirk covered at ** https://sqlite.org/forum/info/e5b20e1feb37a19a. **/ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT void * sqlite3_wasm_ptr_to_sqlite3_free(void){ return (void*)sqlite3_free; } @@ -1647,7 +1677,7 @@ void * sqlite3_wasm_ptr_to_sqlite3_free(void){ ** the virtual FS fails. In builds compiled without SQLITE_ENABLE_WASMFS ** defined, SQLITE_NOTFOUND is returned without side effects. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_init_wasmfs(const char *zMountPoint){ static backend_t pOpfs = 0; if( !zMountPoint || !*zMountPoint ) zMountPoint = "/opfs"; @@ -1667,7 +1697,7 @@ int sqlite3_wasm_init_wasmfs(const char *zMountPoint){ return pOpfs ? 0 : SQLITE_NOMEM; } #else -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_init_wasmfs(const char *zUnused){ //emscripten_console_warn("WASMFS OPFS is not compiled in."); if(zUnused){/*unused*/} @@ -1677,51 +1707,51 @@ int sqlite3_wasm_init_wasmfs(const char *zUnused){ #if SQLITE_WASM_TESTS -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_test_intptr(int * p){ return *p = *p * 2; } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT void * sqlite3_wasm_test_voidptr(void * p){ return p; } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int64_t sqlite3_wasm_test_int64_max(void){ return (int64_t)0x7fffffffffffffff; } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int64_t sqlite3_wasm_test_int64_min(void){ return ~sqlite3_wasm_test_int64_max(); } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int64_t sqlite3_wasm_test_int64_times2(int64_t x){ return x * 2; } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT void sqlite3_wasm_test_int64_minmax(int64_t * min, int64_t *max){ *max = sqlite3_wasm_test_int64_max(); *min = sqlite3_wasm_test_int64_min(); /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int64_t sqlite3_wasm_test_int64ptr(int64_t * p){ /*printf("sqlite3_wasm_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ return *p = *p * 2; } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT void sqlite3_wasm_test_stack_overflow(int recurse){ if(recurse) sqlite3_wasm_test_stack_overflow(recurse); } /* For testing the 'string:dealloc' whwasmutil.xWrap() conversion. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT char * sqlite3_wasm_test_str_hello(int fail){ char * s = fail ? 0 : (char *)sqlite3_malloc(6); if(s){ @@ -1732,4 +1762,4 @@ char * sqlite3_wasm_test_str_hello(int fail){ } #endif /* SQLITE_WASM_TESTS */ -#undef SQLITE_WASM_KEEP +#undef SQLITE_WASM_EXPORT diff --git a/ext/wasm/api/sqlite3-worker1.c-pp.js b/ext/wasm/api/sqlite3-worker1.c-pp.js index 9e9c3ac426..f260422309 100644 --- a/ext/wasm/api/sqlite3-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1.c-pp.js @@ -31,20 +31,20 @@ - `sqlite3.dir`, if set, treats the given directory name as the directory from which `sqlite3.js` will be loaded. */ -"use strict"; -(()=>{ //#if target=es6-bundler-friendly - importScripts('sqlite3.js'); +import {default as sqlite3InitModule} from './sqlite3-bundler-friendly.mjs'; //#else - const urlParams = new URL(self.location.href).searchParams; +"use strict"; +{ + const urlParams = globalThis.location + ? new URL(self.location.href).searchParams + : new URLSearchParams(); let theJs = 'sqlite3.js'; if(urlParams.has('sqlite3.dir')){ theJs = urlParams.get('sqlite3.dir') + '/' + theJs; } //console.warn("worker1 theJs =",theJs); importScripts(theJs); +} //#endif - sqlite3InitModule().then((sqlite3)=>{ - sqlite3.initWorker1API(); - }); -})(); +sqlite3InitModule().then(sqlite3 => sqlite3.initWorker1API()); diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index e50210206f..4899799412 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -45,8 +45,8 @@ Intended usage: ``` - self.WhWasmUtilInstaller(appObject); - delete self.WhWasmUtilInstaller; + globalThis.WhWasmUtilInstaller(appObject); + delete globalThis.WhWasmUtilInstaller; ``` Its global-scope symbol is intended only to provide an easy way to @@ -171,7 +171,7 @@ https://fossil.wanderinghorse.net/r/jaccwabbyt/file/common/whwasmutil.js */ -self.WhWasmUtilInstaller = function(target){ +globalThis.WhWasmUtilInstaller = function(target){ 'use strict'; if(undefined===target.bigIntEnabled){ target.bigIntEnabled = !!self['BigInt64Array']; @@ -2194,7 +2194,7 @@ self.WhWasmUtilInstaller = function(target){ Error handling is up to the caller, who may attach a `catch()` call to the promise. */ -self.WhWasmUtilInstaller.yawl = function(config){ +globalThis.WhWasmUtilInstaller.yawl = function(config){ const wfetch = ()=>fetch(config.uri, {credentials: 'same-origin'}); const wui = this; const finalThen = function(arg){ @@ -2240,4 +2240,4 @@ self.WhWasmUtilInstaller.yawl = function(config){ .then(finalThen); }; return loadWasm; -}.bind(self.WhWasmUtilInstaller)/*yawl()*/; +}.bind(globalThis.WhWasmUtilInstaller)/*yawl()*/; diff --git a/ext/wasm/demo-worker1-promiser.js b/ext/wasm/demo-worker1-promiser.js index ef955403c5..e8d3d4e5ac 100644 --- a/ext/wasm/demo-worker1-promiser.js +++ b/ext/wasm/demo-worker1-promiser.js @@ -9,7 +9,7 @@ * May you share freely, never taking more than you give. *********************************************************************** - + Demonstration of the sqlite3 Worker API #1 Promiser: a Promise-based proxy for for the sqlite3 Worker #1 API. */ @@ -81,7 +81,7 @@ }); logHtml('', "Sending 'open' message and waiting for its response before continuing..."); - + await wtest('open', { filename: dbFilename, simulateError: 0 /* if true, fail the 'open' */, diff --git a/ext/wasm/demo-worker1.js b/ext/wasm/demo-worker1.js index cc63f3a7cc..f70179c5e8 100644 --- a/ext/wasm/demo-worker1.js +++ b/ext/wasm/demo-worker1.js @@ -62,7 +62,7 @@ return this.queue.shift(); } }; - + const testCount = ()=>{ logHtml("","Total test count:",T.counter+". Total time =",(performance.now() - startTime),"ms"); }; @@ -74,7 +74,7 @@ (ev.workerRespondTime - ev.workerReceivedTime),"ms.", "Round-trip event time =", (performance.now() - ev.departureTime),"ms.", - (evd.errorClass ? ev.message : "")//, JSON.stringify(evd) + (evd.errorClass ? evd.message : "")//, JSON.stringify(evd) ); }; diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index 7073c24b78..3f99ad5a5b 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -70,7 +70,8 @@ STRIP_K1.js := $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js) \ $(sqlite3-worker1-bundler-friendly.js) $(sqlite3-worker1-promiser-bundler-friendly.js) # STRIP_K2.js = list of JS files which need to be passed through # $(bin.stripcomments) with two -k flags. -STRIP_K2.js := $(sqlite3.js) $(sqlite3.mjs) $(sqlite3-bundler-friendly.mjs) +STRIP_K2.js := $(sqlite3.js) $(sqlite3.mjs) \ + $(sqlite3-bundler-friendly.mjs) $(sqlite3-node.mjs) ######################################################################## # dist: create the end-user deliverable archive. # diff --git a/ext/wasm/example_extra_init.c b/ext/wasm/example_extra_init.c new file mode 100644 index 0000000000..b91d757cd5 --- /dev/null +++ b/ext/wasm/example_extra_init.c @@ -0,0 +1,23 @@ +/* +** If the canonical build process finds the file +** sqlite3_wasm_extra_init.c in the main wasm build directory, it +** arranges to include that file in the build of sqlite3.wasm and +** defines SQLITE_EXTRA_INIT=sqlite3_wasm_extra_init. +** +** The C file must define the function sqlite3_wasm_extra_init() with +** this signature: +** +** int sqlite3_wasm_extra_init(const char *) +** +** and the sqlite3 library will call it with an argument of NULL one +** time during sqlite3_initialize(). If it returns non-0, +** initialization of the library will fail. +*/ + +#include "sqlite3.h" +#include + +int sqlite3_wasm_extra_init(const char *z){ + fprintf(stderr,"%s: %s()\n", __FILE__, __func__); + return 0; +} diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 7facd7e9e5..cbe6ab3518 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -29,7 +29,7 @@ fiddle.emcc-flags = \ --minify 0 \ -sALLOW_TABLE_GROWTH \ -sABORTING_MALLOC \ - -sSTRICT_JS \ + -sSTRICT_JS=0 \ -sENVIRONMENT=web,worker \ -sMODULARIZE \ -sDYNAMIC_EXECUTION=0 \ diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index d4ec719fb5..bde7d051e8 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -19,7 +19,7 @@ */ 'use strict'; -self.Jaccwabyt = function StructBinderFactory(config){ +globalThis.Jaccwabyt = function StructBinderFactory(config){ /* ^^^^ it is recommended that clients move that object into wherever they'd like to have it and delete the self-held copy ("self" being the global window or worker object). This API does not require the diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index cd0fdb027c..0d88049b95 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -23,7 +23,7 @@
Downloading...
- +
** ** [[SQLITE_DBCONFIG_DQS_DML]] -**
SQLITE_DBCONFIG_DQS_DML +**
SQLITE_DBCONFIG_DQS_DML
**
The SQLITE_DBCONFIG_DQS_DML option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DML statements ** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The @@ -2387,7 +2407,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_DQS_DDL]] -**
SQLITE_DBCONFIG_DQS_DDL +**
SQLITE_DBCONFIG_DQS_DDL
**
The SQLITE_DBCONFIG_DQS option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DDL statements, ** such as CREATE TABLE and CREATE INDEX. The @@ -2396,7 +2416,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] -**
SQLITE_DBCONFIG_TRUSTED_SCHEMA +**
SQLITE_DBCONFIG_TRUSTED_SCHEMA
**
The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to ** assume that database schemas are untainted by malicious content. ** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite @@ -2416,7 +2436,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] -**
SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +**
SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
**
The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly ** created database file to have a schema format version number (the 4-byte @@ -2425,7 +2445,7 @@ struct sqlite3_mem_methods { ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, ** newly created databases are generally not understandable by SQLite versions ** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there -** is now scarcely any need to generated database files that are compatible +** is now scarcely any need to generate database files that are compatible ** all the way back to version 3.0.0, and so this setting is of little ** practical use, but is provided so that SQLite can continue to claim the ** ability to generate new database files that are compatible with version @@ -2436,6 +2456,38 @@ struct sqlite3_mem_methods { ** not considered a bug since SQLite versions 3.3.0 and earlier do not support ** either generated columns or decending indexes. **
+** +** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] +**
SQLITE_DBCONFIG_STMT_SCANSTATUS
+**
The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in +** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears +** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() +** statistics. For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is prepared and when it +** is stepped. The flag is set (collection of statistics is enabled) +** by default. This option takes two arguments: an integer and a pointer to +** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the statement scanstatus option. If the second argument +** is not NULL, then the value of the statement scanstatus setting after +** processing the first argument is written into the integer that the second +** argument points to. +**
+** +** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] +**
SQLITE_DBCONFIG_REVERSE_SCANORDER
+**
The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order +** in which tables and indexes are scanned so that the scans start at the end +** and work toward the beginning rather than starting at the beginning and +** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the +** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** two arguments which are an integer and a pointer to an integer. The first +** argument is 1, 0, or -1 to enable, disable, or leave unchanged the +** reverse scan order flag, respectively. If the second argument is not NULL, +** then 0 or 1 is written into the integer that the second argument points to +** depending on if the reverse scan order flag is set after processing the +** first argument. +**
+** ** */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ @@ -2456,7 +2508,9 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ +#define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -6201,6 +6255,13 @@ void sqlite3_activate_cerod( ** of the default VFS is not implemented correctly, or not implemented at ** all, then the behavior of sqlite3_sleep() may deviate from the description ** in the previous paragraphs. +** +** If a negative argument is passed to sqlite3_sleep() the results vary by +** VFS and operating system. Some system treat a negative argument as an +** instruction to sleep forever. Others understand it to mean do not sleep +** at all. ^In SQLite version 3.42.0 and later, a negative +** argument passed into sqlite3_sleep() is changed to zero before it is relayed +** down into the xSleep method of the VFS. */ int sqlite3_sleep(int); @@ -9564,18 +9625,28 @@ int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]
SQLITE_VTAB_INNOCUOUS
**
Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a ** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS ** flag unless absolutely necessary. **
+** +** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]
SQLITE_VTAB_USES_ALL_SCHEMAS
+**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** instruct the query planner to begin at least a read transaction on +** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the +** virtual table is used. +**
** */ #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 #define SQLITE_VTAB_INNOCUOUS 2 #define SQLITE_VTAB_DIRECTONLY 3 +#define SQLITE_VTAB_USES_ALL_SCHEMAS 4 /* ** CAPI3REF: Determine The Virtual Table Conflict Policy diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 59f33eb3fa..bf46c20ff7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -224,8 +224,8 @@ #endif /* -** WAL mode depends on atomic aligned 32-bit loads and stores in a few -** places. The following macros try to make this explicit. +** A few places in the code require atomic load/store of aligned +** integer values. */ #ifndef __has_extension # define __has_extension(x) 0 /* compatibility with non-clang compilers */ @@ -281,15 +281,22 @@ #endif /* -** A macro to hint to the compiler that a function should not be +** Macros to hint to the compiler that a function should or should not be ** inlined. */ #if defined(__GNUC__) # define SQLITE_NOINLINE __attribute__((noinline)) +# define SQLITE_INLINE __attribute__((always_inline)) inline #elif defined(_MSC_VER) && _MSC_VER>=1310 # define SQLITE_NOINLINE __declspec(noinline) +# define SQLITE_INLINE __forceinline #else # define SQLITE_NOINLINE +# define SQLITE_INLINE +#endif +#if defined(SQLITE_COVERAGE_TEST) || defined(__STRICT_ANSI__) +# undef SQLITE_INLINE +# define SQLITE_INLINE #endif /* @@ -1775,7 +1782,7 @@ struct sqlite3 { #define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ /* result set is empty */ #define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommit 0x00000400 /* READ UNCOMMITTED in shared-cache */ +#define SQLITE_StmtScanStatus 0x00000400 /* Enable stmt_scanstats() counters */ #define SQLITE_NoCkptOnClose 0x00000800 /* No checkpoint on close()/DETACH */ #define SQLITE_ReverseOrder 0x00001000 /* Reverse unordered SELECTs */ #define SQLITE_RecTriggers 0x00002000 /* Enable recursive triggers */ @@ -1801,6 +1808,7 @@ struct sqlite3 { /* DELETE, or UPDATE and return */ /* the count using a callback. */ #define SQLITE_CorruptRdOnly HI(0x00002) /* Prohibit writes due to error */ +#define SQLITE_ReadUncommit HI(0x00004) /* READ UNCOMMITTED in shared-cache */ /* Flags used by the Pragma noop_update enhancement */ #define SQLITE_NoopUpdate HI(0x0001000) /* UPDATE operations are no-ops */ @@ -1860,6 +1868,7 @@ struct sqlite3 { /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ #define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ #define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ +#define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -2331,6 +2340,7 @@ struct VTable { sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ u8 bConstraint; /* True if constraints are supported */ + u8 bAllSchemas; /* True if might use any attached schema */ u8 eVtabRisk; /* Riskiness of allowing hacker access */ int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ @@ -3362,7 +3372,7 @@ struct NameContext { #define NC_HasAgg 0x000010 /* One or more aggregate functions seen */ #define NC_IdxExpr 0x000020 /* True if resolving columns of CREATE INDEX */ #define NC_SelfRef 0x00002e /* Combo: PartIdx, isCheck, GenCol, and IdxExpr */ -#define NC_VarSelect 0x000040 /* A correlated subquery has been seen */ +#define NC_Subquery 0x000040 /* A subquery has been seen */ #define NC_UEList 0x000080 /* True if uNC.pEList is used */ #define NC_UAggInfo 0x000100 /* True if uNC.pAggInfo is used */ #define NC_UUpsert 0x000200 /* True if uNC.pUpsert is used */ @@ -3681,6 +3691,7 @@ struct IndexedExpr { int iIdxCur; /* The index cursor */ int iIdxCol; /* The index column that contains value of pExpr */ u8 bMaybeNullRow; /* True if we need an OP_IfNullRow check */ + u8 aff; /* Affinity of the pExpr expression */ IndexedExpr *pIENext; /* Next in a list of all indexed expressions */ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS const char *zIdxName; /* Name of index, used only for bytecode comments */ @@ -3732,6 +3743,9 @@ struct Parse { u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ +#endif +#ifdef SQLITE_DEBUG + u8 ifNotExists; /* Might be true if IF NOT EXISTS. Assert()s only */ #endif int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ @@ -4193,6 +4207,7 @@ struct Walker { struct CoveringIndexCheck *pCovIdxCk; /* Check for covering index */ SrcItem *pSrcItem; /* A single FROM clause item */ DbFixer *pFix; /* See sqlite3FixSelect() */ + Mem *aMem; /* See sqlite3BtreeCursorHint() */ } u; }; @@ -4462,6 +4477,8 @@ int sqlite3CantopenError(int); # define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) # define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) # define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) +# define sqlite3JsonId1(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x42) +# define sqlite3JsonId2(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x46) #else # define sqlite3Toupper(x) toupper((unsigned char)(x)) # define sqlite3Isspace(x) isspace((unsigned char)(x)) @@ -4471,6 +4488,8 @@ int sqlite3CantopenError(int); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) # define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') +# define sqlite3JsonId1(x) (sqlite3IsIdChar(x)&&(x)<'0') +# define sqlite3JsonId2(x) sqlite3IsIdChar(x) #endif int sqlite3IsIdChar(u8); @@ -4664,6 +4683,10 @@ void sqlite3ReleaseTempReg(Parse*,int); int sqlite3GetTempRange(Parse*,int); void sqlite3ReleaseTempRange(Parse*,int,int); void sqlite3ClearTempRegCache(Parse*); +void sqlite3TouchRegister(Parse*,int); +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) +int sqlite3FirstAvailableRegister(Parse*,int); +#endif #ifdef SQLITE_DEBUG int sqlite3NoTempsInRange(Parse*,int,int); #endif @@ -4814,7 +4837,7 @@ Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, Expr*,ExprList*,u32,Expr*); void sqlite3SelectDelete(sqlite3*, Select*); Table *sqlite3SrcListLookup(Parse*, SrcList*); -int sqlite3IsReadOnly(Parse*, Table*, int); +int sqlite3IsReadOnly(Parse*, Table*, Trigger*); void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*); @@ -5353,10 +5376,7 @@ int sqlite3VtabCallDestroy(sqlite3*, int, const char *); int sqlite3VtabBegin(sqlite3 *, VTable *); FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); -#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ - && !defined(SQLITE_OMIT_VIRTUALTABLE) - void sqlite3VtabUsesAllSchemas(sqlite3_index_info*); -#endif +void sqlite3VtabUsesAllSchemas(Parse*); sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); @@ -5603,4 +5623,10 @@ int sqlite3KvvfsInit(void); sqlite3_uint64 sqlite3Hwtime(void); #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define IS_STMT_SCANSTATUS(db) (db->flags & SQLITE_StmtScanStatus) +#else +# define IS_STMT_SCANSTATUS(db) 0 +#endif + #endif /* SQLITEINT_H */ diff --git a/src/test1.c b/src/test1.c index 46a1b1cb9b..8b0592b10d 100644 --- a/src/test1.c +++ b/src/test1.c @@ -16,6 +16,13 @@ #include "sqliteInt.h" #if SQLITE_OS_WIN # include "os_win.h" +# include +#else +# include +# if defined(__APPLE__) +# include +# include +# endif #endif #include "vdbeInt.h" @@ -2384,6 +2391,31 @@ static int SQLITE_TCLAPI vfsCurrentTimeInt64( return TCL_OK; } +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Usage: create_null_module DB NAME +*/ +static int SQLITE_TCLAPI test_create_null_module( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + char *zName; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zName = Tcl_GetString(objv[2]); + + sqlite3_create_module(db, zName, 0, 0); + return TCL_OK; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + #ifdef SQLITE_ENABLE_SNAPSHOT /* ** Usage: sqlite3_snapshot_get DB DBNAME @@ -8283,6 +8315,7 @@ static int SQLITE_TCLAPI test_sqlite3_db_config( { "DQS_DML", SQLITE_DBCONFIG_DQS_DML }, { "DQS_DDL", SQLITE_DBCONFIG_DQS_DDL }, { "LEGACY_FILE_FORMAT", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, + { "STMT_SCANSTATUS", SQLITE_DBCONFIG_STMT_SCANSTATUS }, }; int i; int v = 0; @@ -8674,6 +8707,40 @@ static int SQLITE_TCLAPI test_autovacuum_pages( return TCL_OK; } +/* +** Usage: number_of_cores +** +** Return a guess at the number of available cores available on the +** processor on which this process is running. +*/ +static int SQLITE_TCLAPI guess_number_of_cores( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + unsigned int nCore = 1; +#if SQLITE_OS_WIN + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + nCore = (unsigned int)sysinfo.dwNumberOfProcessors; +#elif defined(__APPLE__) + int nm[2]; + size_t len = 4; + nm[0] = CTL_HW; nm[1] = HW_AVAILCPU; + sysctl(nm, 2, &nCore, &len, NULL, 0); + if( nCore<1 ){ + nm[1] = HW_NCPU; + sysctl(nm, 2, &nCore, &len, NULL, 0); + } +#else + nCore = sysconf(_SC_NPROCESSORS_ONLN); +#endif + if( nCore<=0 ) nCore = 1; + Tcl_SetObjResult(interp, Tcl_NewIntObj((int)nCore)); + return SQLITE_OK; +} + /* ** Register commands with the TCL interpreter. @@ -8975,6 +9042,10 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "test_write_db", test_write_db, 0 }, { "sqlite3_register_cksumvfs", test_register_cksumvfs, 0 }, { "sqlite3_unregister_cksumvfs", test_unregister_cksumvfs, 0 }, + { "number_of_cores", guess_number_of_cores, 0 }, +#ifndef SQLITE_OMIT_VIRTUALTABLE + { "create_null_module", test_create_null_module, 0 }, +#endif }; static int bitmask_size = sizeof(Bitmask)*8; static int longdouble_size = sizeof(LONGDOUBLE_TYPE); diff --git a/src/test_syscall.c b/src/test_syscall.c index 947f9a9d9d..3cd1034d3f 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -110,15 +110,15 @@ static int ts_stat(const char *zPath, struct stat *p); static int ts_fstat(int fd, struct stat *p); static int ts_ftruncate(int fd, off_t n); static int ts_fcntl(int fd, int cmd, ... ); -static int ts_read(int fd, void *aBuf, size_t nBuf); -static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off); +static ssize_t ts_read(int fd, void *aBuf, size_t nBuf); +static ssize_t ts_pread(int fd, void *aBuf, size_t nBuf, off_t off); /* Note: pread64() and pwrite64() actually use off64_t as the type on their ** last parameter. But that datatype is not defined on many systems ** (ex: Mac, OpenBSD). So substitute a likely equivalent: sqlite3_uint64 */ -static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off); -static int ts_write(int fd, const void *aBuf, size_t nBuf); -static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off); -static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off); +static ssize_t ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off); +static ssize_t ts_write(int fd, const void *aBuf, size_t nBuf); +static ssize_t ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off); +static ssize_t ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); static void *ts_mmap(void *, size_t, int, int, int, off_t); @@ -211,7 +211,7 @@ static int tsErrno(const char *zFunc){ /* ** A wrapper around tsIsFail(). If tsIsFail() returns non-zero, set the ** value of errno before returning. -*/ +*/ static int tsIsFailErrno(const char *zFunc){ if( tsIsFail() ){ errno = tsErrno(zFunc); @@ -313,7 +313,7 @@ static int ts_fcntl(int fd, int cmd, ... ){ /* ** A wrapper around read(). */ -static int ts_read(int fd, void *aBuf, size_t nBuf){ +static ssize_t ts_read(int fd, void *aBuf, size_t nBuf){ if( tsIsFailErrno("read") ){ return -1; } @@ -323,7 +323,7 @@ static int ts_read(int fd, void *aBuf, size_t nBuf){ /* ** A wrapper around pread(). */ -static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){ +static ssize_t ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){ if( tsIsFailErrno("pread") ){ return -1; } @@ -333,7 +333,7 @@ static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){ /* ** A wrapper around pread64(). */ -static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){ +static ssize_t ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){ if( tsIsFailErrno("pread64") ){ return -1; } @@ -343,7 +343,7 @@ static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){ /* ** A wrapper around write(). */ -static int ts_write(int fd, const void *aBuf, size_t nBuf){ +static ssize_t ts_write(int fd, const void *aBuf, size_t nBuf){ if( tsIsFailErrno("write") ){ if( tsErrno("write")==EINTR ) orig_write(fd, aBuf, nBuf/2); return -1; @@ -354,7 +354,7 @@ static int ts_write(int fd, const void *aBuf, size_t nBuf){ /* ** A wrapper around pwrite(). */ -static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){ +static ssize_t ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){ if( tsIsFailErrno("pwrite") ){ return -1; } @@ -364,7 +364,7 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){ /* ** A wrapper around pwrite64(). */ -static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off){ +static ssize_t ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off){ if( tsIsFailErrno("pwrite64") ){ return -1; } diff --git a/src/trigger.c b/src/trigger.c index 02d8540237..bcb2132f0b 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -202,6 +202,7 @@ void sqlite3BeginTrigger( }else{ assert( !db->init.busy ); sqlite3CodeVerifySchema(pParse, iDb); + VVA_ONLY( pParse->ifNotExists = 1; ) } goto trigger_cleanup; } @@ -983,7 +984,7 @@ static void codeReturningTrigger( } sqlite3ExprListDelete(db, sSelect.pEList); pNew = sqlite3ExpandReturning(pParse, pReturning->pReturnEL, pTab); - if( !db->mallocFailed ){ + if( pParse->nErr==0 ){ NameContext sNC; memset(&sNC, 0, sizeof(sNC)); if( pReturning->nRetCol==0 ){ @@ -1452,6 +1453,9 @@ u32 sqlite3TriggerColmask( Trigger *p; assert( isNew==1 || isNew==0 ); + if( IsView(pTab) ){ + return 0xffffffff; + } for(p=pTrigger; p; p=p->pNext){ if( p->op==op && (tr_tm&p->tr_tm) diff --git a/src/update.c b/src/update.c index fb065b9ec4..e28cca8e2d 100644 --- a/src/update.c +++ b/src/update.c @@ -408,7 +408,7 @@ void sqlite3Update( if( sqlite3ViewGetColumnNames(pParse, pTab) ){ goto update_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto update_cleanup; } @@ -738,12 +738,22 @@ void sqlite3Update( /* Begin the database scan. ** ** Do not consider a single-pass strategy for a multi-row update if - ** there are any triggers or foreign keys to process, or rows may - ** be deleted as a result of REPLACE conflict handling. Any of these - ** things might disturb a cursor being used to scan through the table - ** or index, causing a single-pass approach to malfunction. */ + ** there is anything that might disrupt the cursor being used to do + ** the UPDATE: + ** (1) This is a nested UPDATE + ** (2) There are triggers + ** (3) There are FOREIGN KEY constraints + ** (4) There are REPLACE conflict handlers + ** (5) There are subqueries in the WHERE clause + */ flags = WHERE_ONEPASS_DESIRED; - if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){ + if( !pParse->nested + && !pTrigger + && !hasFK + && !chngKey + && !bReplace + && (sNC.ncFlags & NC_Subquery)==0 + ){ flags |= WHERE_ONEPASS_MULTIROW; } pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,0,0,flags,iIdxCur); diff --git a/src/util.c b/src/util.c index 632d317e31..3d4e014382 100644 --- a/src/util.c +++ b/src/util.c @@ -670,13 +670,15 @@ int sqlite3Int64ToText(i64 v, char *zOut){ } i = sizeof(zTemp)-2; zTemp[sizeof(zTemp)-1] = 0; - do{ - zTemp[i--] = (x%10) + '0'; + while( 1 /*exit-by-break*/ ){ + zTemp[i] = (x%10) + '0'; x = x/10; - }while( x ); - if( v<0 ) zTemp[i--] = '-'; - memcpy(zOut, &zTemp[i+1], sizeof(zTemp)-1-i); - return sizeof(zTemp)-2-i; + if( x==0 ) break; + i--; + }; + if( v<0 ) zTemp[--i] = '-'; + memcpy(zOut, &zTemp[i], sizeof(zTemp)-i); + return sizeof(zTemp)-1-i; } /* @@ -841,7 +843,9 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ u = u*16 + sqlite3HexToInt(z[k]); } memcpy(pOut, &u, 8); - return (z[k]==0 && k-i<=16) ? 0 : 2; + if( k-i>16 ) return 2; + if( z[k]!=0 ) return 1; + return 0; }else #endif /* SQLITE_OMIT_HEX_INTEGER */ { @@ -877,7 +881,7 @@ int sqlite3GetInt32(const char *zNum, int *pValue){ u32 u = 0; zNum += 2; while( zNum[0]=='0' ) zNum++; - for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){ + for(i=0; i<8 && sqlite3Isxdigit(zNum[i]); i++){ u = u*16 + sqlite3HexToInt(zNum[i]); } if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){ diff --git a/src/vdbe.c b/src/vdbe.c index ba80f0a32d..5944f22431 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -683,8 +683,10 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ }else if( p->flags & MEM_Real ){ h += sqlite3VdbeIntValue(p); }else if( p->flags & (MEM_Str|MEM_Blob) ){ - h += p->n; - if( p->flags & MEM_Zero ) h += p->u.nZero; + /* All strings have the same hash and all blobs have the same hash, + ** though, at least, those hashes are different from each other and + ** from NULL. */ + h += 4093 + (p->flags & (MEM_Str|MEM_Blob)); } } return h; @@ -734,6 +736,7 @@ int sqlite3VdbeExec( Mem *pOut = 0; /* Output operand */ #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) u64 *pnCycle = 0; + int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0; #endif /*** INSERT STACK UNION HERE ***/ @@ -798,13 +801,17 @@ int sqlite3VdbeExec( assert( pOp>=aOp && pOp<&aOp[p->nOp]); nVmStep++; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + +#if defined(VDBE_PROFILE) pOp->nExec++; pnCycle = &pOp->nCycle; -# ifdef VDBE_PROFILE - if( sqlite3NProfileCnt==0 ) -# endif + if( sqlite3NProfileCnt==0 ) *pnCycle -= sqlite3Hwtime(); +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( bStmtScanStatus ){ + pOp->nExec++; + pnCycle = &pOp->nCycle; *pnCycle -= sqlite3Hwtime(); + } #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -2619,6 +2626,12 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ ** (0x01) bit. SQLITE_FLOAT is the 0x02 bit. SQLITE_TEXT is 0x04. ** SQLITE_BLOB is 0x08. SQLITE_NULL is 0x10. ** +** WARNING: This opcode does not reliably distinguish between NULL and REAL +** when P1>=0. If the database contains a NaN value, this opcode will think +** that the datatype is REAL when it should be NULL. When P1<0 and the value +** is already stored in register P3, then this opcode does reliably +** distinguish between NULL and REAL. The problem only arises then P1>=0. +** ** Take the jump to address P2 if and only if the datatype of the ** value determined by P1 and P3 corresponds to one of the bits in the ** P5 bitmask. @@ -2732,7 +2745,7 @@ case OP_IfNullRow: { /* jump */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - if( ALWAYS(pC) && pC->nullRow ){ + if( pC && pC->nullRow ){ sqlite3VdbeMemSetNull(aMem + pOp->p3); goto jump_to_p2; } @@ -3227,7 +3240,7 @@ case OP_Affinity: { }else{ pIn1->u.r = (double)pIn1->u.i; pIn1->flags |= MEM_Real; - pIn1->flags &= ~MEM_Int; + pIn1->flags &= ~(MEM_Int|MEM_Str); } } REGISTER_TRACE((int)(pIn1-aMem), pIn1); @@ -5002,6 +5015,7 @@ case OP_SeekScan: { /* ncycle */ break; } nStep--; + pC->cacheStatus = CACHE_STALE; rc = sqlite3BtreeNext(pC->uc.pCursor, 0); if( rc ){ if( rc==SQLITE_DONE ){ @@ -7654,6 +7668,7 @@ case OP_AggFinal: { } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); + REGISTER_TRACE((int)(pMem-aMem), pMem); break; } @@ -8797,8 +8812,10 @@ default: { /* This is really OP_Noop, OP_Explain */ *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); pnCycle = 0; #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) - *pnCycle += sqlite3Hwtime(); - pnCycle = 0; + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; + } #endif /* The following code adds nothing to the actual functionality diff --git a/src/vdbe.h b/src/vdbe.h index 3caf1f7872..d28837f944 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -403,4 +403,8 @@ void sqlite3VdbeScanStatusCounters(Vdbe*, int, int, int); void sqlite3VdbePrintOp(FILE*, int, VdbeOp*); #endif +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr); +#endif + #endif /* SQLITE_VDBE_H */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 476b6a2adf..d8fcda96df 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -271,7 +271,7 @@ int sqlite3_value_type(sqlite3_value* pVal){ SQLITE_NULL, /* 0x1f (not possible) */ SQLITE_FLOAT, /* 0x20 INTREAL */ SQLITE_NULL, /* 0x21 (not possible) */ - SQLITE_TEXT, /* 0x22 INTREAL + TEXT */ + SQLITE_FLOAT, /* 0x22 INTREAL + TEXT */ SQLITE_NULL, /* 0x23 (not possible) */ SQLITE_FLOAT, /* 0x24 (not possible) */ SQLITE_NULL, /* 0x25 (not possible) */ @@ -1337,9 +1337,9 @@ static const void *columnName( assert( db!=0 ); n = sqlite3_column_count(pStmt); if( N=0 ){ + u8 prior_mallocFailed = db->mallocFailed; N += useType*n; sqlite3_mutex_enter(db->mutex); - assert( db->mallocFailed==0 ); #ifndef SQLITE_OMIT_UTF16 if( useUtf16 ){ ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); @@ -1351,7 +1351,8 @@ static const void *columnName( /* A malloc may have failed inside of the _text() call. If this ** is the case, clear the mallocFailed flag and return NULL. */ - if( db->mallocFailed ){ + assert( db->mallocFailed==0 || db->mallocFailed==1 ); + if( db->mallocFailed > prior_mallocFailed ){ sqlite3OomClear(db); ret = 0; } @@ -2138,15 +2139,24 @@ int sqlite3_stmt_scanstatus_v2( void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; - ScanStatus *pScan; + VdbeOp *aOp = p->aOp; + int nOp = p->nOp; + ScanStatus *pScan = 0; int idx; + if( p->pFrame ){ + VdbeFrame *pFrame; + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + aOp = pFrame->aOp; + nOp = pFrame->nOp; + } + if( iScan<0 ){ int ii; if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){ i64 res = 0; - for(ii=0; iinOp; ii++){ - res += p->aOp[ii].nCycle; + for(ii=0; iiaddrLoop>0 ){ - *(sqlite3_int64*)pOut = p->aOp[pScan->addrLoop].nExec; + *(sqlite3_int64*)pOut = aOp[pScan->addrLoop].nExec; }else{ *(sqlite3_int64*)pOut = -1; } @@ -2180,7 +2190,7 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_NVISIT: { if( pScan->addrVisit>0 ){ - *(sqlite3_int64*)pOut = p->aOp[pScan->addrVisit].nExec; + *(sqlite3_int64*)pOut = aOp[pScan->addrVisit].nExec; }else{ *(sqlite3_int64*)pOut = -1; } @@ -2202,7 +2212,7 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_EXPLAIN: { if( pScan->addrExplain ){ - *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + *(const char**)pOut = aOp[ pScan->addrExplain ].p4.z; }else{ *(const char**)pOut = 0; } @@ -2210,7 +2220,7 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_SELECTID: { if( pScan->addrExplain ){ - *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + *(int*)pOut = aOp[ pScan->addrExplain ].p1; }else{ *(int*)pOut = -1; } @@ -2218,7 +2228,7 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_PARENTID: { if( pScan->addrExplain ){ - *(int*)pOut = p->aOp[ pScan->addrExplain ].p2; + *(int*)pOut = aOp[ pScan->addrExplain ].p2; }else{ *(int*)pOut = -1; } @@ -2236,18 +2246,18 @@ int sqlite3_stmt_scanstatus_v2( if( iIns==0 ) break; if( iIns>0 ){ while( iIns<=iEnd ){ - res += p->aOp[iIns].nCycle; + res += aOp[iIns].nCycle; iIns++; } }else{ int iOp; - for(iOp=0; iOpnOp; iOp++){ - Op *pOp = &p->aOp[iOp]; + for(iOp=0; iOpp1!=iEnd ) continue; if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ continue; } - res += p->aOp[iOp].nCycle; + res += aOp[iOp].nCycle; } } } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index eb473b38dd..d97e21ea80 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -443,10 +443,10 @@ void sqlite3ExplainBreakpoint(const char *z1, const char *z2){ */ int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ int addr = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) +#if !defined(SQLITE_DEBUG) /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. ** But omit them (for performance) during production builds */ - if( pParse->explain==2 ) + if( pParse->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { char *zMsg; @@ -820,6 +820,8 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ Op *pOp; Parse *pParse = p->pParse; int *aLabel = pParse->aLabel; + + assert( pParse->db->mallocFailed==0 ); /* tag-20230419-1 */ p->readOnly = 1; p->bIsReader = 0; pOp = &p->aOp[p->nOp-1]; @@ -879,6 +881,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ); assert( ADDR(pOp->p2)<-pParse->nLabel ); + assert( aLabel!=0 ); /* True because of tag-20230419-1 */ pOp->p2 = aLabel[ADDR(pOp->p2)]; } break; @@ -1122,18 +1125,20 @@ void sqlite3VdbeScanStatus( LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ - sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); - ScanStatus *aNew; - aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); - if( aNew ){ - ScanStatus *pNew = &aNew[p->nScan++]; - memset(pNew, 0, sizeof(ScanStatus)); - pNew->addrExplain = addrExplain; - pNew->addrLoop = addrLoop; - pNew->addrVisit = addrVisit; - pNew->nEst = nEst; - pNew->zName = sqlite3DbStrDup(p->db, zName); - p->aScan = aNew; + if( IS_STMT_SCANSTATUS(p->db) ){ + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanStatus *pNew = &aNew[p->nScan++]; + memset(pNew, 0, sizeof(ScanStatus)); + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } } } @@ -1150,20 +1155,22 @@ void sqlite3VdbeScanStatusRange( int addrStart, int addrEnd ){ - ScanStatus *pScan = 0; - int ii; - for(ii=p->nScan-1; ii>=0; ii--){ - pScan = &p->aScan[ii]; - if( pScan->addrExplain==addrExplain ) break; - pScan = 0; - } - if( pScan ){ - if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; - for(ii=0; iiaAddrRange); ii+=2){ - if( pScan->aAddrRange[ii]==0 ){ - pScan->aAddrRange[ii] = addrStart; - pScan->aAddrRange[ii+1] = addrEnd; - break; + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; + for(ii=0; iiaAddrRange); ii+=2){ + if( pScan->aAddrRange[ii]==0 ){ + pScan->aAddrRange[ii] = addrStart; + pScan->aAddrRange[ii+1] = addrEnd; + break; + } } } } @@ -1180,19 +1187,21 @@ void sqlite3VdbeScanStatusCounters( int addrLoop, int addrVisit ){ - ScanStatus *pScan = 0; - int ii; - for(ii=p->nScan-1; ii>=0; ii--){ - pScan = &p->aScan[ii]; - if( pScan->addrExplain==addrExplain ) break; - pScan = 0; - } - if( pScan ){ - pScan->addrLoop = addrLoop; - pScan->addrVisit = addrVisit; + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + pScan->addrLoop = addrLoop; + pScan->addrVisit = addrVisit; + } } } -#endif +#endif /* defined(SQLITE_ENABLE_STMT_SCANSTATUS) */ /* @@ -1616,7 +1625,7 @@ VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ /* Return the most recently added opcode */ -VdbeOp * sqlite3VdbeGetLastOp(Vdbe *p){ +VdbeOp *sqlite3VdbeGetLastOp(Vdbe *p){ return sqlite3VdbeGetOp(p, p->nOp - 1); } @@ -3338,6 +3347,8 @@ int sqlite3VdbeHalt(Vdbe *p){ db->flags &= ~(u64)SQLITE_DeferFKs; sqlite3CommitInternalChanges(db); } + }else if( p->rc==SQLITE_SCHEMA && db->nVdbeActive>1 ){ + p->nChange = 0; }else{ sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; @@ -3658,9 +3669,9 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ #ifdef SQLITE_ENABLE_NORMALIZE sqlite3DbFree(db, p->zNormSql); { - DblquoteStr *pThis, *pNext; - for(pThis=p->pDblStr; pThis; pThis=pNext){ - pNext = pThis->pNextStr; + DblquoteStr *pThis, *pNxt; + for(pThis=p->pDblStr; pThis; pThis=pNxt){ + pNxt = pThis->pNextStr; sqlite3DbFree(db, pThis); } } @@ -5287,6 +5298,20 @@ int sqlite3NotPureFunc(sqlite3_context *pCtx){ return 1; } +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +/* +** This Walker callback is used to help verify that calls to +** sqlite3BtreeCursorHint() with opcode BTREE_HINT_RANGE have +** byte-code register values correctly initialized. +*/ +int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_REGISTER ){ + assert( (pWalker->u.aMem[pExpr->iTable].flags & MEM_Undefined)==0 ); + } + return WRC_Continue; +} +#endif /* SQLITE_ENABLE_CURSOR_HINTS && SQLITE_DEBUG */ + #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored @@ -5349,6 +5374,16 @@ void sqlite3VdbePreUpdateHook( PreUpdate preupdate; const char *zTbl = pTab->zName; static const u8 fakeSortOrder = 0; +#ifdef SQLITE_DEBUG + int nRealCol; + if( pTab->tabFlags & TF_WithoutRowid ){ + nRealCol = sqlite3PrimaryKeyIndex(pTab)->nColumn; + }else if( pTab->tabFlags & TF_HasVirtual ){ + nRealCol = pTab->nNVCol; + }else{ + nRealCol = pTab->nCol; + } +#endif assert( db->pPreUpdate==0 ); memset(&preupdate, 0, sizeof(PreUpdate)); @@ -5365,8 +5400,8 @@ void sqlite3VdbePreUpdateHook( assert( pCsr!=0 ); assert( pCsr->eCurType==CURTYPE_BTREE ); - assert( pCsr->nField==pTab->nCol - || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1) + assert( pCsr->nField==nRealCol + || (pCsr->nField==nRealCol+1 && op==SQLITE_DELETE && iReg==-1) ); preupdate.v = v; diff --git a/src/vdbemem.c b/src/vdbemem.c index be52062d55..d3cd55ba9f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -157,6 +157,7 @@ int sqlite3VdbeMemValidStrRep(Mem *p){ char *z; int i, j, incr; if( (p->flags & MEM_Str)==0 ) return 1; + if( p->db && p->db->mallocFailed ) return 1; if( p->flags & MEM_Term ){ /* Insure that the string is properly zero-terminated. Pay particular ** attention to the case where p->n is odd */ @@ -439,7 +440,7 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ vdbeMemRenderNum(nByte, pMem->z, pMem); assert( pMem->z!=0 ); - assert( pMem->n==sqlite3Strlen30NN(pMem->z) ); + assert( pMem->n==(int)sqlite3Strlen30NN(pMem->z) ); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); @@ -1483,8 +1484,11 @@ static int valueFromFunction( if( pList ) nVal = pList->nExpr; assert( !ExprHasProperty(p, EP_IntValue) ); pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pFunc==0 ) return SQLITE_OK; +#endif assert( pFunc ); - if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) ){ return SQLITE_OK; @@ -1519,16 +1523,11 @@ static int valueFromFunction( }else{ sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8); assert( rc==SQLITE_OK ); - assert( enc==pVal->enc - || (pVal->flags & MEM_Str)==0 - || db->mallocFailed ); -#if 0 /* Not reachable except after a prior failure */ rc = sqlite3VdbeChangeEncoding(pVal, enc); - if( rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal) ){ + if( NEVER(rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal)) ){ rc = SQLITE_TOOBIG; pCtx->pParse->nErr++; } -#endif } value_from_function_out: @@ -1592,6 +1591,13 @@ static int valueFromExpr( rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); testcase( rc!=SQLITE_OK ); if( *ppVal ){ +#ifdef SQLITE_ENABLE_STAT4 + rc = ExpandBlob(*ppVal); +#else + /* zero-blobs only come from functions, not literal values. And + ** functions are only processed under STAT4 */ + assert( (ppVal[0][0].flags & MEM_Zero)==0 ); +#endif sqlite3VdbeMemCast(*ppVal, aff, enc); sqlite3ValueApplyAffinity(*ppVal, affinity, enc); } diff --git a/src/vtab.c b/src/vtab.c index 2dd92762b7..3477d67ce3 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -610,7 +610,9 @@ static int vtabCallConstructor( sCtx.pPrior = db->pVtabCtx; sCtx.bDeclared = 0; db->pVtabCtx = &sCtx; + pTab->nTabRef++; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + sqlite3DeleteTable(db, pTab); db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); assert( sCtx.pTab==pTab ); @@ -1329,6 +1331,10 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){ p->pVTable->eVtabRisk = SQLITE_VTABRISK_High; break; } + case SQLITE_VTAB_USES_ALL_SCHEMAS: { + p->pVTable->bAllSchemas = 1; + break; + } default: { rc = SQLITE_MISUSE_BKPT; break; diff --git a/src/wal.c b/src/wal.c index f269f6ec49..0f7072eacd 100644 --- a/src/wal.c +++ b/src/wal.c @@ -748,19 +748,40 @@ static void walChecksumBytes( assert( nByte>=8 ); assert( (nByte&0x00000007)==0 ); assert( nByte<=65536 ); + assert( nByte%4==0 ); - if( nativeCksum ){ - do { - s1 += *aData++ + s2; - s2 += *aData++ + s1; - }while( aDataexplain!=2 ){ + if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){ Table *pTab = pIdx->pTable; const char *zSep = ""; char *zText = 0; @@ -892,7 +892,8 @@ static SQLITE_NOINLINE void constructAutomaticIndex( char *zNotUsed; /* Extra space on the end of pIdx */ Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ - u8 sentWarning = 0; /* True if a warnning has been issued */ + u8 sentWarning = 0; /* True if a warning has been issued */ + u8 useBloomFilter = 0; /* True to also add a Bloom filter */ Expr *pPartial = 0; /* Partial Index Expression */ int iContinue = 0; /* Jump here to skip excluded rows */ SrcItem *pTabItem; /* FROM clause term being indexed */ @@ -962,7 +963,11 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** original table changes and the index and table cannot both be used ** if they go out of sync. */ - extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + if( IsView(pTable) ){ + extraCols = ALLBITS; + }else{ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + } mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); @@ -998,6 +1003,16 @@ static SQLITE_NOINLINE void constructAutomaticIndex( assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */ pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; n++; + if( ALWAYS(pX->pLeft!=0) + && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT + ){ + /* TUNING: only use a Bloom filter on an automatic index + ** if one or more key columns has the ability to hold numeric + ** values, since strings all have the same hash in the Bloom + ** filter implementation and hence a Bloom filter on a text column + ** is not usually helpful. */ + useBloomFilter = 1; + } } } } @@ -1030,7 +1045,8 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "for %s", pTable->zName)); - if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){ + sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel); pLevel->regFilter = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter); } @@ -1123,6 +1139,10 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( Vdbe *v = pParse->pVdbe; /* VDBE under construction */ WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */ int iCur; /* Cursor for table getting the filter */ + IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */ + + saved_pIdxEpr = pParse->pIdxEpr; + pParse->pIdxEpr = 0; assert( pLoop!=0 ); assert( v!=0 ); @@ -1179,9 +1199,8 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( int r1 = sqlite3GetTempRange(pParse, n); int jj; for(jj=0; jjaiColumn[jj]; assert( pIdx->pTable==pItem->pTab ); - sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj); } sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); sqlite3ReleaseTempRange(pParse, r1, n); @@ -1212,6 +1231,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( } }while( iLevel < pWInfo->nLevel ); sqlite3VdbeJumpHere(v, addrOnce); + pParse->pIdxEpr = saved_pIdxEpr; } @@ -1467,6 +1487,9 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); } } + if( pTab->u.vtab.p->bAllSchemas ){ + sqlite3VtabUsesAllSchemas(pParse); + } sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; return rc; @@ -1510,6 +1533,7 @@ static int whereKeyStats( assert( pRec!=0 ); assert( pIdx->nSample>0 ); assert( pRec->nField>0 ); + /* Do a binary search to find the first sample greater than or equal ** to pRec. If pRec contains a single field, the set of samples to search @@ -1555,7 +1579,12 @@ static int whereKeyStats( ** it is extended to two fields. The duplicates that this creates do not ** cause any problems. */ - nField = MIN(pRec->nField, pIdx->nSample); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nField = pIdx->nKeyCol; + }else{ + nField = pIdx->nColumn; + } + nField = MIN(pRec->nField, nField); iCol = 0; iSample = pIdx->nSample * nField; do{ @@ -1991,7 +2020,7 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); assert( pLower || pUpper ); #endif - assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); @@ -4092,8 +4121,6 @@ int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ return pHidden->eDistinct; } -#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ - && !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** Cause the prepared statement that is associated with a call to ** xBestIndex to potentially use all schemas. If the statement being @@ -4103,9 +4130,7 @@ int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ ** ** This is used by the (built-in) sqlite_dbpage virtual table. */ -void sqlite3VtabUsesAllSchemas(sqlite3_index_info *pIdxInfo){ - HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; - Parse *pParse = pHidden->pParse; +void sqlite3VtabUsesAllSchemas(Parse *pParse){ int nDb = pParse->db->nDb; int i; for(i=0; iisOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } + if( pWInfo->pSelect->pOrderBy + && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){ + pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr; + } }else{ pWInfo->revMask = pFrom->revLoop; if( pWInfo->nOBSat<=0 ){ @@ -5694,6 +5722,9 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( p->iIdxCur = iIdxCur; p->iIdxCol = i; p->bMaybeNullRow = bMaybeNullRow; + if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ + p->aff = pIdx->zColAff[i]; + } #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS p->zIdxName = pIdx->zName; #endif @@ -5951,22 +5982,45 @@ WhereInfo *sqlite3WhereBegin( } if( pParse->nErr ) goto whereBeginError; - /* Special case: WHERE terms that do not refer to any tables in the join - ** (constant expressions). Evaluate each such term, and jump over all the - ** generated code if the result is not true. + /* The False-WHERE-Term-Bypass optimization: ** - ** Do not do this if the expression contains non-deterministic functions - ** that are not within a sub-select. This is not strictly required, but - ** preserves SQLite's legacy behaviour in the following two cases: + ** If there are WHERE terms that are false, then no rows will be output, + ** so skip over all of the code generated here. ** - ** FROM ... WHERE random()>0; -- eval random() once per row - ** FROM ... WHERE (SELECT random())>0; -- eval random() once overall + ** Conditions: + ** + ** (1) The WHERE term must not refer to any tables in the join. + ** (2) The term must not come from an ON clause on the + ** right-hand side of a LEFT or FULL JOIN. + ** (3) The term must not come from an ON clause, or there must be + ** no RIGHT or FULL OUTER joins in pTabList. + ** (4) If the expression contains non-deterministic functions + ** that are not within a sub-select. This is not required + ** for correctness but rather to preserves SQLite's legacy + ** behaviour in the following two cases: + ** + ** WHERE random()>0; -- eval random() once per row + ** WHERE (SELECT random())>0; -- eval random() just once overall + ** + ** Note that the Where term need not be a constant in order for this + ** optimization to apply, though it does need to be constant relative to + ** the current subquery (condition 1). The term might include variables + ** from outer queries so that the value of the term changes from one + ** invocation of the current subquery to the next. */ for(ii=0; iinBase; ii++){ - WhereTerm *pT = &sWLB.pWC->a[ii]; + WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */ + Expr *pX; /* The expression of pT */ if( pT->wtFlags & TERM_VIRTUAL ) continue; - if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){ - sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL); + pX = pT->pExpr; + assert( pX!=0 ); + assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) ); + if( pT->prereqAll==0 /* Conditions (1) and (2) */ + && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */ + && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */ + && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 ) + ){ + sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL); pT->wtFlags |= TERM_CODED; } } @@ -6209,7 +6263,7 @@ WhereInfo *sqlite3WhereBegin( assert( n<=pTab->nCol ); } #ifdef SQLITE_ENABLE_CURSOR_HINTS - if( pLoop->u.btree.pIndex!=0 ){ + if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); }else #endif @@ -6667,7 +6721,8 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ k = pLevel->addrBody + 1; #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeAddopTrace ){ - printf("TRANSLATE opcodes in range %d..%d\n", k, last-1); + printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n", + pLevel->iTabCur, pLevel->iIdxCur, k, last-1); } /* Proof that the "+1" on the k value above is safe */ pOp = sqlite3VdbeGetOp(v, k - 1); diff --git a/src/wherecode.c b/src/wherecode.c index 860acb44cf..5441fb17d3 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -111,9 +111,9 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ /* ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was -** defined at compile-time. If it is not a no-op, a single OP_Explain opcode -** is added to the output to describe the table scan strategy in pLevel. +** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG +** was defined at compile-time. If it is not a no-op, a single OP_Explain +** opcode is added to the output to describe the table scan strategy in pLevel. ** ** If an OP_Explain opcode is added to the VM, its address is returned. ** Otherwise, if no OP_Explain is coded, zero is returned. @@ -125,8 +125,8 @@ int sqlite3WhereExplainOneScan( u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ int ret = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( sqlite3ParseToplevel(pParse)->explain==2 ) +#if !defined(SQLITE_DEBUG) + if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { SrcItem *pItem = &pTabList->a[pLevel->iFrom]; @@ -292,27 +292,29 @@ void sqlite3WhereAddScanStatus( WhereLevel *pLvl, /* Level to add scanstatus() entry for */ int addrExplain /* Address of OP_Explain (or 0) */ ){ - const char *zObj = 0; - WhereLoop *pLoop = pLvl->pWLoop; - int wsFlags = pLoop->wsFlags; - int viaCoroutine = 0; + if( IS_STMT_SCANSTATUS( sqlite3VdbeDb(v) ) ){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + int wsFlags = pLoop->wsFlags; + int viaCoroutine = 0; - if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ - zObj = pLoop->u.btree.pIndex->zName; - }else{ - zObj = pSrclist->a[pLvl->iFrom].zName; - viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; - } - sqlite3VdbeScanStatus( - v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj - ); - - if( viaCoroutine==0 ){ - if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ - sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pSrclist->a[pLvl->iFrom].zName; + viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; } - if( wsFlags & WHERE_INDEXED ){ - sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); + + if( viaCoroutine==0 ){ + if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + } + if( wsFlags & WHERE_INDEXED ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + } } } } @@ -1009,11 +1011,12 @@ static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){ */ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ int rc = WRC_Continue; + int reg; struct CCurHint *pHint = pWalker->u.pCCurHint; if( pExpr->op==TK_COLUMN ){ if( pExpr->iTable!=pHint->iTabCur ){ - int reg = ++pWalker->pParse->nMem; /* Register for column value */ - sqlite3ExprCode(pWalker->pParse, pExpr, reg); + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); pExpr->op = TK_REGISTER; pExpr->iTable = reg; }else if( pHint->pIdx!=0 ){ @@ -1021,15 +1024,12 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn); assert( pExpr->iColumn>=0 ); } - }else if( pExpr->op==TK_AGG_FUNCTION ){ - /* An aggregate function in the WHERE clause of a query means this must - ** be a correlated sub-query, and expression pExpr is an aggregate from - ** the parent context. Do not walk the function arguments in this case. - ** - ** todo: It should be possible to replace this node with a TK_REGISTER - ** expression, as the result of the expression must be stored in a - ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ + }else if( pExpr->pAggInfo ){ rc = WRC_Prune; + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); + pExpr->op = TK_REGISTER; + pExpr->iTable = reg; } return rc; } @@ -1131,7 +1131,7 @@ static void codeCursorHint( } if( pExpr!=0 ){ sWalker.xExprCallback = codeCursorHintFixExpr; - sqlite3WalkExpr(&sWalker, pExpr); + if( pParse->nErr==0 ) sqlite3WalkExpr(&sWalker, pExpr); sqlite3VdbeAddOp4(v, OP_CursorHint, (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, (const char*)pExpr, P4_EXPR); @@ -1925,7 +1925,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** guess. */ addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, (pIdx->aiRowLogEst[0]+9)/10); - if( pRangeStart ){ + if( pRangeStart || pRangeEnd ){ sqlite3VdbeChangeP5(v, 1); sqlite3VdbeChangeP2(v, addrSeekScan, sqlite3VdbeCurrentAddr(v)+1); addrSeekScan = 0; @@ -1966,16 +1966,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( assert( pLevel->p2==0 ); if( pRangeEnd ){ Expr *pRight = pRangeEnd->pExpr->pRight; - if( addrSeekScan ){ - /* For a seek-scan that has a range on the lowest term of the index, - ** we have to make the top of the loop be code that sets the end - ** condition of the range. Otherwise, the OP_SeekScan might jump - ** over that initialization, leaving the range-end value set to the - ** range-start value, resulting in a wrong answer. - ** See ticket 5981a8c041a3c2f3 (2021-11-02). - */ - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - } + assert( addrSeekScan==0 ); codeExprOrVector(pParse, pRight, regBase+nEq, nTop); whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); if( (pRangeEnd->wtFlags & TERM_VNULL)==0 @@ -2009,7 +2000,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff); /* Top of the loop body */ - if( pLevel->p2==0 ) pLevel->p2 = sqlite3VdbeCurrentAddr(v); + pLevel->p2 = sqlite3VdbeCurrentAddr(v); /* Check if the index cursor is past the end of the range. */ if( nConstraint ){ diff --git a/src/whereexpr.c b/src/whereexpr.c index 4f86753bf7..d02c8ccfc7 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1856,9 +1856,12 @@ void sqlite3WhereTabFuncArgs( pRhs = sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs); - if( pItem->fg.jointype & (JT_LEFT|JT_LTORJ) ){ + if( pItem->fg.jointype & (JT_LEFT|JT_RIGHT) ){ + testcase( pItem->fg.jointype & JT_LEFT ); /* testtag-20230227a */ + testcase( pItem->fg.jointype & JT_RIGHT ); /* testtag-20230227b */ joinType = EP_OuterON; }else{ + testcase( pItem->fg.jointype & JT_LTORJ ); /* testtag-20230227c */ joinType = EP_InnerON; } sqlite3SetJoinExpr(pTerm, pItem->iCursor, joinType); diff --git a/src/window.c b/src/window.c index 56de38ba39..a8081aa244 100644 --- a/src/window.c +++ b/src/window.c @@ -785,6 +785,7 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ } /* no break */ deliberate_fall_through + case TK_IF_NULL_ROW: case TK_AGG_FUNCTION: case TK_COLUMN: { int iCol = -1; diff --git a/test/aggfault.test b/test/aggfault.test new file mode 100644 index 0000000000..7c16b039ce --- /dev/null +++ b/test/aggfault.test @@ -0,0 +1,43 @@ +# 2023 March 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix aggfault + + +do_execsql_test 1 { + CREATE TABLE t1(x); + CREATE INDEX t1x ON t1(x, x=0); +} +faultsim_save_and_close + +do_faultsim_test 2 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM sqlite_schema } +} -body { + execsql { + SELECT * FROM t1 AS a1 WHERE ( + SELECT count(x AND 0=a1.x) FROM t1 GROUP BY abs(1) + ) AND x=( + SELECT * FROM t1 AS a1 + WHERE (SELECT count(x IS 1 AND a1.x=0) + FROM t1 + GROUP BY abs(1)) AND x=0 + ); + } +} -test { + faultsim_test_result {0 {}} +} + + +finish_test diff --git a/test/altertab.test b/test/altertab.test index 6d8347ec1e..9cc43e14de 100644 --- a/test/altertab.test +++ b/test/altertab.test @@ -981,4 +981,22 @@ do_catchsql_test 32.0 { ALTER TABLE t1 RENAME TO x; } {1 {error in trigger r1: no tables specified}} +# 2023-04-13 https://sqlite.org/forum/forumpost/ff3840145a +# +reset_db +do_execsql_test 33.0 { + CREATE TABLE t1(a TEXT); + INSERT INTO t1(a) VALUES('abc'),('def'),(NULL); + CREATE TABLE t2(b TEXT); + CREATE TRIGGER r3 AFTER INSERT ON t1 BEGIN + UPDATE t2 SET (b,a)=(SELECT 1) FROM t1 JOIN t2 ON (SELECT * FROM (SELECT a)); + END; +} +do_catchsql_test 33.1 { + ALTER TABLE t1 RENAME COLUMN a TO b; +} {1 {error in trigger r3 after rename: no such column: a}} +do_execsql_test 33.2 { + SELECT quote(a) FROM t1 ORDER BY +a; +} {NULL 'abc' 'def'} + finish_test diff --git a/test/analyze3.test b/test/analyze3.test index 7469c537c0..322d6fb775 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -736,4 +736,17 @@ do_execsql_test 7.2 { ANALYZE sqlite_master; } +# 2023-04-22 https://sqlite.org/forum/info/6c118daad0f1f5ef +# Case differences in the sqlite_stat4.idx field should not matter. +# +reset_db +do_execsql_test 8.0 { + CREATE TABLE t1(a PRIMARY KEY, v) WITHOUT ROWID; + ANALYZE sqlite_schema; + INSERT INTO sqlite_stat1 VALUES('t1','t1','1 1'); + INSERT INTO sqlite_stat4 VALUES('t1','t1','1','0','0',X'021b76657273696f6e'); + INSERT INTO sqlite_stat4 VALUES('T1','T1','1','0','0',X'021b76657273696f6e'); + ANALYZE sqlite_schema; +} {} + finish_test diff --git a/test/analyzeE.test b/test/analyzeE.test index 733b79367f..2547daa1c8 100644 --- a/test/analyzeE.test +++ b/test/analyzeE.test @@ -239,4 +239,56 @@ do_execsql_test analyzeE-4.11 { SELECT * FROM t1 WHERE a<1900 AND c=123 } {/SCAN t1/} +# 2023-03-23 https://sqlite.org/forum/forumpost/dc4854437b +# +reset_db +do_execsql_test analyzeE-5.0 { + PRAGMA encoding = 'UTF-16'; + CREATE TABLE t0 (c1 TEXT); + INSERT INTO t0 VALUES (''); + CREATE INDEX i0 ON t0(c1); + ANALYZE; + SELECT * FROM t0 WHERE t0.c1 BETWEEN '' AND (ABS('')); +} {{}} + +# 2023-03-24 https://sqlite.org/forum/forumpost/bc39e531e5 +# +reset_db +do_execsql_test analyzeE-6.0 { + CREATE TABLE t1(x); + CREATE INDEX i1 ON t1(x,x,x,x,x||2); + CREATE INDEX i2 ON t1(1<2); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t1(x) SELECT x FROM c; + ANALYZE; +} {} +do_execsql_test analyzeE-6.1 { + SELECT count(*)>1 FROM sqlite_stat4 WHERE idx='i2' AND neq='1000 1'; +} 1 +do_execsql_test analyzeE-6.2 { + SELECT count(*) FROM sqlite_stat4 WHERE idx='i2' AND neq<>'1000 1'; +} 0 +do_execsql_test analyzeE-6.3 { + SELECT count(*)>1 FROM sqlite_stat4 WHERE idx='i1' AND neq='1 1 1 1 1 1'; +} 1 +do_execsql_test analyzeE-6.4 { + SELECT count(*) FROM sqlite_stat4 WHERE idx='i1' AND neq<>'1 1 1 1 1 1'; +} 0 + +# 2023-03-25 https://sqlite.org/forum/forumpost/5275207102 +# Correctly expand zeroblobs while processing STAT4 information +# during query planning. +# +reset_db +do_execsql_test analyzeE-7.0 { + CREATE TABLE t1(a TEXT COLLATE binary); + CREATE INDEX t1x ON t1(a); + INSERT INTO t1(a) VALUES(0),('apple'),(NULL),(''),('banana'); + ANALYZE; + SELECT format('(%s)',a) FROM t1 WHERE t1.a > CAST(zeroblob(5) AS TEXT); +} {(0) (apple) (banana)} +do_execsql_test analyzeE-7.1 { + SELECT format('(%s)',a) FROM t1 WHERE t1.a <= CAST(zeroblob(5) AS TEXT); +} {()} + finish_test diff --git a/test/autoindex1.test b/test/autoindex1.test index 4b290c7961..08bd9f74e6 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -194,6 +194,7 @@ do_eqp_test autoindex1-501 { QUERY PLAN |--SCAN t501 `--CORRELATED LIST SUBQUERY xxxxxx + |--BLOOM FILTER ON t502 (y=?) `--SEARCH t502 USING AUTOMATIC COVERING INDEX (y=?) } do_eqp_test autoindex1-502 { diff --git a/test/autoindex3.test b/test/autoindex3.test index 824a82973b..3da7a70586 100644 --- a/test/autoindex3.test +++ b/test/autoindex3.test @@ -86,6 +86,7 @@ do_eqp_test 220 { } { QUERY PLAN |--SEARCH v USING INDEX ve (e>?) + |--BLOOM FILTER ON u (b=?) `--SEARCH u USING AUTOMATIC COVERING INDEX (b=?) } diff --git a/test/backup.test b/test/backup.test index 1572ded554..ad1a383a08 100644 --- a/test/backup.test +++ b/test/backup.test @@ -977,5 +977,28 @@ do_test backup-11.1 { sqlite3_backup B db1 main db2 temp B finish } {SQLITE_OK} +db1 close +db2 close + +#------------------------------------------------------------------------- +do_test backup-12.1 { + sqlite3 db1 :memory: + sqlite3 db2 :memory: + db1 eval { + PRAGMA page_size = 8192; + CREATE TABLE t1(x); + } + db2 eval { + PRAGMA page_size = 1024; + CREATE TABLE t2(x); + } + + sqlite3_backup B db1 main db2 temp + B step 100 + B finish +} {SQLITE_READONLY} + + + finish_test diff --git a/test/bloom1.test b/test/bloom1.test index 1846e4d63f..151f364ae0 100644 --- a/test/bloom1.test +++ b/test/bloom1.test @@ -99,6 +99,90 @@ do_eqp_test 2.1 { | |--SCAN transit | `--SEARCH objs USING COVERING INDEX objs_cspo (o=? AND p=?) `--SCAN transit -} +} + +# 2023-02-28 +# https://sqlite.org/forum/forumpost/0846211821 +# +# Bloom filter gives an incorrect result if the collating sequence is +# anything other than binary. +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t0(x TEXT COLLATE rtrim); + INSERT INTO t0(x) VALUES ('a'), ('b'), ('c'); + CREATE VIEW v0(y) AS SELECT DISTINCT x FROM t0; + SELECT count(*) FROM t0, v0 WHERE x='b '; +} 3 +do_eqp_test 3.2 { + SELECT count(*) FROM t0, v0 WHERE x='b '; +} { + QUERY PLAN + |--CO-ROUTINE v0 + | |--SCAN t0 + | `--USE TEMP B-TREE FOR DISTINCT + |--SCAN v0 + `--SEARCH t0 USING AUTOMATIC PARTIAL COVERING INDEX (x=?) +} +# ^^^^^--- The key feature in the previous result is that no Bloom filter +# is used. In the following, a Bloom filter is used because the data type +# is INT instead of TEXT. +do_execsql_test 3.3 { + CREATE TABLE t1(x INT COLLATE rtrim); + INSERT INTO t1(x) VALUES ('a'), ('b'), ('c'); + CREATE VIEW v1(y) AS SELECT DISTINCT x FROM t1; + SELECT count(*) FROM t1, v1 WHERE x='b '; +} 3 +do_eqp_test 3.4 { + SELECT count(*) FROM t1, v1 WHERE x='b '; +} { + QUERY PLAN + |--CO-ROUTINE v1 + | |--SCAN t1 + | `--USE TEMP B-TREE FOR DISTINCT + |--SCAN v1 + |--BLOOM FILTER ON t1 (x=?) + `--SEARCH t1 USING AUTOMATIC PARTIAL COVERING INDEX (x=?) +} + +# 2023-03-14 +# https://sqlite.org/forum/forumpost/d47a0e8e3a +# https://sqlite.org/forum/forumpost/2e427099d5 +# +# Both reports are for the same problem - using a Bloom filter on an +# expression index can cause issues. +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(x TEXT, y INT, z TEXT); + INSERT INTO t1(rowid,x,y,z) VALUES(12,'aa','bb','aa'); + CREATE INDEX i1x ON t1(1 IS true,z); + CREATE TABLE t0(x TEXT); + INSERT INTO t0(rowid,x) VALUES(4,'aa'); + ANALYZE sqlite_schema; + INSERT INTO sqlite_stat1 VALUES('t0',NULL,'20'); + INSERT INTO sqlite_stat1 VALUES('t1','i1x','18 18 2'); + ANALYZE sqlite_schema; +} +do_execsql_test 4.2 { + SELECT * FROM t0 NATURAL JOIN t1 WHERE z=t1.x; +} {aa bb aa} +do_execsql_test 4.3 { + DROP TABLE t0; + CREATE TABLE t0(a TEXT); + INSERT INTO t0 VALUES ('xyz'); + CREATE INDEX t0x ON t0(a IS FALSE) WHERE false; + DROP TABLE t1; + CREATE TABLE t1(b INT); + INSERT INTO t1 VALUES('aaa'),('bbb'),('ccc'),('ddd'),(NULL); + CREATE TABLE t2(c REAL); + INSERT INTO t2 VALUES(7); + ANALYZE; + CREATE INDEX t2x ON t2(true IN ()); +} +do_execsql_test 4.4 { + SELECT * FROM t0 LEFT JOIN t1 LEFT JOIN t2 ON (b NOTNULL)==(c IN ()) WHERE c; +} {xyz {} 7.0} + finish_test diff --git a/test/corrupt2.test b/test/corrupt2.test index f97a526ef5..96d28490aa 100644 --- a/test/corrupt2.test +++ b/test/corrupt2.test @@ -99,7 +99,7 @@ do_test corrupt2-1.4 { # of MemPage.nFree catchsql {PRAGMA quick_check} db2 } {0 {{*** in database main *** -Page 1: free space corruption}}} +Tree 1 page 1: free space corruption}}} do_test corrupt2-1.5 { db2 close @@ -120,7 +120,7 @@ do_test corrupt2-1.5 { sqlite3 db2 corrupt.db catchsql {PRAGMA quick_check} db2 } {0 {{*** in database main *** -Page 1: free space corruption}}} +Tree 1 page 1: free space corruption}}} db2 close # Corrupt a database by having 2 indices of the same name: @@ -248,8 +248,8 @@ do_test corrupt2-5.1 { } set result } {{*** in database main *** -On tree page 2 cell 0: 2nd reference to page 10 -Page 4 is never used}} +Tree 11 page 2 cell 0: 2nd reference to page 10 +Page 4: never used}} db2 close @@ -591,7 +591,7 @@ do_test 14.2 { do_execsql_test 14.3 { PRAGMA integrity_check; } {{*** in database main *** -Main freelist: size is 3 but should be 2}} +Freelist: size is 3 but should be 2}} # Use 2 of the free pages on the free-list. # @@ -603,7 +603,7 @@ do_execsql_test 14.4 { do_execsql_test 14.5 { PRAGMA integrity_check; } {{*** in database main *** -Main freelist: size is 1 but should be 0}} +Freelist: size is 1 but should be 0}} finish_test diff --git a/test/corrupt3.test b/test/corrupt3.test index 7a2e174ead..691302f7af 100644 --- a/test/corrupt3.test +++ b/test/corrupt3.test @@ -82,7 +82,7 @@ do_test corrupt3-1.8 { PRAGMA integrity_check } } {0 {{*** in database main *** -On tree page 2 cell 0: 2nd reference to page 3}}} +Tree 2 page 2 cell 0: 2nd reference to page 3}}} # Change the pointer for the first page of the overflow # change to be a non-existant page. @@ -100,8 +100,8 @@ do_test corrupt3-1.10 { PRAGMA integrity_check } } {0 {{*** in database main *** -On tree page 2 cell 0: invalid page number 4 -Page 3 is never used}}} +Tree 2 page 2 cell 0: invalid page number 4 +Page 3: never used}}} do_test corrupt3-1.11 { db close hexio_write test.db 2044 [hexio_render_int32 0] @@ -115,7 +115,7 @@ do_test corrupt3-1.12 { PRAGMA integrity_check } } {0 {{*** in database main *** -On tree page 2 cell 0: overflow list length is 0 but should be 1 -Page 3 is never used}}} +Tree 2 page 2 cell 0: overflow list length is 0 but should be 1 +Page 3: never used}}} finish_test diff --git a/test/corrupt7.test b/test/corrupt7.test index aa66cc7ece..b62515b784 100644 --- a/test/corrupt7.test +++ b/test/corrupt7.test @@ -70,14 +70,14 @@ do_test corrupt7-2.1 { sqlite3 db test.db db eval {PRAGMA integrity_check(1)} } {{*** in database main *** -On tree page 2 cell 15: Offset 65457 out of range 945..1020}} +Tree 2 page 2 cell 15: Offset 65457 out of range 945..1020}} do_test corrupt7-2.2 { db close hexio_write test.db 1062 04 sqlite3 db test.db db eval {PRAGMA integrity_check(1)} } {{*** in database main *** -On tree page 2 cell 15: Offset 1201 out of range 945..1020}} +Tree 2 page 2 cell 15: Offset 1201 out of range 945..1020}} # The code path that was causing the buffer overrun that this test # case was checking for was removed. diff --git a/test/corruptC.test b/test/corruptC.test index a56abeec72..f5733a8186 100644 --- a/test/corruptC.test +++ b/test/corruptC.test @@ -98,7 +98,7 @@ do_test corruptC-2.1 { sqlite3 db test.db catchsql {PRAGMA integrity_check} } {0 {{*** in database main *** -Page 3: free space corruption}}} +Tree 3 page 3: free space corruption}}} # test that a corrupt content offset size is handled (seed 5649) # @@ -165,7 +165,7 @@ do_test corruptC-2.5 { catchsql {BEGIN; UPDATE t2 SET y='abcdef-uvwxyz'; ROLLBACK;} catchsql {PRAGMA integrity_check} } {0 {{*** in database main *** -On tree page 4 cell 19: Extends off end of page} {database disk image is malformed}}} +Tree 4 page 4 cell 19: Extends off end of page} {database disk image is malformed}}} # {0 {{*** in database main *** # Corruption detected in cell 710 on page 4 diff --git a/test/corruptD.test b/test/corruptD.test index eb6ccb3fcd..c35388adfb 100644 --- a/test/corruptD.test +++ b/test/corruptD.test @@ -113,7 +113,7 @@ do_test corruptD-1.1.1 { hexio_write test.db [expr 1024+1] FFFF catchsql { PRAGMA quick_check } } {0 {{*** in database main *** -Page 2: free space corruption}}} +Tree 2 page 2: free space corruption}}} do_test corruptD-1.1.2 { incr_change_counter hexio_write test.db [expr 1024+1] [hexio_render_int32 1021] diff --git a/test/corruptI.test b/test/corruptI.test index 12ce446540..65ef376258 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -123,18 +123,13 @@ do_execsql_test 4.0 { set root [db one {SELECT rootpage FROM sqlite_master}] set offset [expr ($root-1) * 65536] -ifcapable oversize_cell_check { - set res {1 {database disk image is malformed}} -} else { - set res {0 {}} -} do_test 4.1 { db close hexio_write test.db [expr $offset + 8 + 2] 0000 hexio_write test.db [expr $offset + 5] 0000 sqlite3 db test.db catchsql { DELETE FROM t1 WHERE a=0 } -} $res +} {1 {database disk image is malformed}} #------------------------------------------------------------------------- diff --git a/test/corruptL.test b/test/corruptL.test index 3a841a8199..65d9821b35 100644 --- a/test/corruptL.test +++ b/test/corruptL.test @@ -1269,11 +1269,14 @@ do_test 15.0 { }]} {} extra_schema_checks 0 -do_execsql_test 15.1 { +do_catchsql_test 15.1 { PRAGMA cell_size_check = 0; UPDATE c1 SET c= NOT EXISTS(SELECT 1 FROM c1 ORDER BY (SELECT 1 FROM c1 ORDER BY a)) +10 WHERE d BETWEEN 4 AND 7; -} {} +} {1 {database disk image is malformed}} extra_schema_checks 1 +do_execsql_test 15.2 { + PRAGMA integrity_check; +} {/in database main/} #------------------------------------------------------------------------- reset_db diff --git a/test/countofview.test b/test/countofview.test index 0ee511ff34..cea6fa4274 100644 --- a/test/countofview.test +++ b/test/countofview.test @@ -51,6 +51,57 @@ do_execsql_test 2.1 { SELECT count(*) FROM v1 GROUP BY y; } {3 3} +# 2023-03-01 dbsqlfuzz ef8623915d843b150c159166ee4548c78cc6895a +# count-of-view should not apply to CTEs. +# +ifcapable progress { + proc progress_stop args {return 1} + db progress 1000 progress_stop + do_catchsql_test 3.1 { + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c) + SELECT count(*) FROM c; + } {1 interrupted} +} + +# 2023-03-07 dbsqlfuzz 23d782160b71c3f8f535ccb2da313dfc8eb8c631 +# +do_execsql_test 4.1 { + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1 VALUES(4,'four'); + CREATE TABLE t2(c INTEGER PRIMARY KEY, d TEXT); + CREATE VIEW t3 AS SELECT a, b FROM t1 UNION ALL SELECT c, d FROM t2; + SELECT count(*) FROM t3 ORDER BY sum(a); +} 1 + +# 2023-03-31 dbsqlfuzz 6a107e3055bd22afab31cfddabc2d9d54fcbaf69 +# Having clauses should disqualify count-of-view +# +reset_db +do_execsql_test 5.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1 VALUES(1,'one'),(4,'four'); + CREATE TABLE t2(c INTEGER PRIMARY KEY, d TEXT); + INSERT INTO t2 VALUES(2,'two'),(5,'five'); + CREATE VIEW t3 AS SELECT a, b FROM t1 UNION ALL SELECT c, d FROM t2; + SELECT count(*) FROM t3 HAVING count(*)>0; +} 4 +do_execsql_test 5.2 { + SELECT count(*) FROM t3 HAVING count(*)>5; +} {} +do_execsql_test 5.3 { + SELECT count(*) FROM t3 HAVING max(b)>'mmm'; +} 4 +do_execsql_test 5.4 { + SELECT count(*) FROM t3 HAVING min(b)>'mmm'; +} {} +do_execsql_test 5.5 { + SELECT count(*) FROM ( + SELECT a, max(b) FROM t1 HAVING a<100 UNION ALL SELECT c, d FROM t2 + ) +} 3 finish_test diff --git a/test/cursorhint.test b/test/cursorhint.test index a3397b8673..47d9f76f39 100644 --- a/test/cursorhint.test +++ b/test/cursorhint.test @@ -159,4 +159,36 @@ do_test 4.6desc { } } {AND(AND(EQ(c0,22),GE(c1,10)),LE(c1,20))} +# 2023-03-24 https://sqlite.org/forum/forumpost/591006b1cc +# +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(x TEXT PRIMARY KEY) WITHOUT ROWID; + CREATE VIEW t2 AS SELECT 0 FROM t1 WHERE x>='a' OR x='1'; + SELECT * FROM t2 RIGHT JOIN t1 ON true; +} +# Additional test case from https://sqlite.org/forum/forumpost/d34ad68c36?t=c +# which is a different way to acces the same problem. +# +do_execsql_test 5.1 { + CREATE TABLE v1 (c1, PRIMARY KEY( c1 )) WITHOUT ROWID; + CREATE VIEW v2 AS SELECT 0 FROM v1 WHERE c1 IS '' OR c1 > ''; + CREATE VIEW v3 AS SELECT 0 FROM v2 JOIN (v2 RIGHT JOIN v1); + CREATE VIEW v4 AS SELECT 0 FROM v3, v3; + SELECT * FROM v3 JOIN v3 AS a0, v4 AS a1, v4 AS a2, v3 AS a3, + v3 AS a4, v4 AS a5 + ORDER BY 1; +} + +# 2023-04-10 https://sqlite.org/forum/forumpost/0b53708c95 +# +do_execsql_test 6.0 { + CREATE TABLE t6(a TEXT UNIQUE, b TEXT); + INSERT INTO t6(a,b) VALUES('uvw','xyz'),('abc','def'); + WITH v1(a) AS (SELECT a COLLATE NOCASE FROM t6) + SELECT v1.a, count(*) FROM t6 LEFT JOIN v1 ON true + GROUP BY 1 + HAVING (SELECT true FROM t6 AS aa LEFT JOIN t6 AS bb ON length(v1.a)>5); +} {abc 2 uvw 2} + finish_test diff --git a/test/delete.test b/test/delete.test index a448e52dd2..214bae6f70 100644 --- a/test/delete.test +++ b/test/delete.test @@ -416,4 +416,28 @@ do_execsql_test delete-11.1 { } {6 2 12 4 18 6 19 23 20 40} +# 2023-03-15 +# https://sqlite.org/forum/forumpost/e61252062c9d286d +# +# When the WHERE clause of a DELETE statement contains a subquery +# which uses the table that is being deleted from and there is a +# short-circuit operator of some kind in the WHERE clause such that +# the subquery might not run right away, then the subquery might +# run after one or more rows have been deleted, which can change +# the result of the subquery, and result in the wrong answer. +# +# Similar problem for UPDATE tested by update-21.4 +# https://sqlite.org/forum/forumpost/0007d1fdb1 +# +reset_db +do_execsql_test delete-12.0 { + CREATE TABLE t0(vkey INTEGER, pkey INTEGER,c1 INTEGER); + INSERT INTO t0 VALUES(2,1,-20),(2,2,NULL),(2,3,0),(8,4,95); + DELETE FROM t0 WHERE NOT ( + (t0.vkey <= t0.c1) AND + (t0.vkey <> (SELECT vkey FROM t0 ORDER BY vkey LIMIT 1 OFFSET 2)) + ); + SELECT * FROM t0; +} {8 4 95} + finish_test diff --git a/test/distinct.test b/test/distinct.test index 446f85bb8e..760b2341f5 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -348,5 +348,61 @@ foreach {tn idx} { } } +# 2023-03-16 +# https://sqlite.org/forum/forumpost/16ce2bb7a639e29b +# ticket c36cdb4afd504dc1 +# ticket 4051a7f931d9ba24 +# ticket d6fd512f50513ab7 +# +do_execsql_test 10.1 { + SELECT DISTINCT + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 + ORDER BY + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x'; +} {1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1} +do_execsql_test 10.2 { + EXPLAIN + SELECT DISTINCT + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 + ORDER BY + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x'; +} {/0 Init 0 /} +do_execsql_test 10.3 { + EXPLAIN CREATE TABLE t2 AS SELECT DISTINCT ':memory:', 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 ORDER BY '%J%j%w%s', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', '%J%j%w%s', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 42e-300, 'unixepoch', 'unixepoch', 'unixepoch' LIMIT 0xda; +} {/0 Init 0/} +do_execsql_test 10.4 { + DROP TABLE IF EXISTS t0; + CREATE TABLE t0 AS SELECT DISTINCT 0xda, 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 0xda-0xda-42e-300, 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0' ORDER BY '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%Y-%m-%d', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', 'lit0', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', 'auto', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', ':memory:', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', ''; + SELECT count(*) FROM t0; +} {1} +do_execsql_test 10.5 { + DROP TABLE IF EXISTS t2; + CREATE TABLE t2 AS SELECT DISTINCT ':memory:', 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0.0*7/0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 ORDER BY '%J%j%w%s', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', '%J%j%w%s', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 42e-300, 'unixepoch', 'unixepoch', 'unixepoch' LIMIT 0xda; + SELECT count(*) FROM t2; +} {1} finish_test diff --git a/test/distinct2.test b/test/distinct2.test index 46eace6f0f..958d9634fd 100644 --- a/test/distinct2.test +++ b/test/distinct2.test @@ -301,4 +301,16 @@ do_execsql_test 3030 { {} 1 a } +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 4010 { + CREATE TABLE t1(a, b COLLATE RTRIM); + INSERT INTO t1 VALUES(1, ''), (2, ' '), (3, ' '); +} +do_execsql_test 4020 { + SELECT b FROM t1 UNION SELECT 1; +} {1 { }} + finish_test diff --git a/test/fkey1.test b/test/fkey1.test index db93be501d..46e7f64a19 100644 --- a/test/fkey1.test +++ b/test/fkey1.test @@ -272,4 +272,15 @@ do_catchsql_test 8.3 { REINDEX; } {1 {database disk image is malformed}} +# 2023-04-13 https://bugs.chromium.org/p/chromium/issues/detail?id=1405220 +# Avoid double-de-quoting of table names when processing foreign keys. +# +reset_db +do_execsql_test 9.1 { + PRAGMA foreign_keys = ON; + CREATE TABLE """1"("""2", """3" PRIMARY KEY); + CREATE TABLE """4"("""5" REFERENCES """1" ON DELETE RESTRICT); + DELETE FROM """1"; +} + finish_test diff --git a/test/fts3join.test b/test/fts3join.test index 3333b1ab42..cbd08b63f2 100644 --- a/test/fts3join.test +++ b/test/fts3join.test @@ -100,6 +100,7 @@ do_eqp_test 4.2 { |--MATERIALIZE rr | `--SCAN ft4 VIRTUAL TABLE INDEX 3: |--SCAN t4 + |--BLOOM FILTER ON rr (docid=?) `--SEARCH rr USING AUTOMATIC COVERING INDEX (docid=?) LEFT-JOIN } diff --git a/test/func8.test b/test/func8.test new file mode 100644 index 0000000000..348dfb7f66 --- /dev/null +++ b/test/func8.test @@ -0,0 +1,64 @@ +# 2023-03-17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# Test cases for SQL functions with names that are the same as join +# keywords: CROSS FULL INNER LEFT NATURAL OUTER RIGHT +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +proc joinx {args} {return [join $args -]} +db func cross {joinx cross} +db func full {joinx full} +db func inner {joinx inner} +db func left {joinx left} +db func natural {joinx natural} +db func outer {joinx outer} +db func right {joinx right} +do_execsql_test func8-100 { + CREATE TABLE cross(cross,full,inner,left,natural,outer,right); + CREATE TABLE full(cross,full,inner,left,natural,outer,right); + CREATE TABLE inner(cross,full,inner,left,natural,outer,right); + CREATE TABLE left(cross,full,inner,left,natural,outer,right); + CREATE TABLE natural(cross,full,inner,left,natural,outer,right); + CREATE TABLE outer(cross,full,inner,left,natural,outer,right); + CREATE TABLE right(cross,full,inner,left,natural,outer,right); + INSERT INTO cross VALUES(1,2,3,4,5,6,7); + INSERT INTO full VALUES(1,2,3,4,5,6,7); + INSERT INTO inner VALUES(1,2,3,4,5,6,7); + INSERT INTO left VALUES(1,2,3,4,5,6,7); + INSERT INTO natural VALUES(1,2,3,4,5,6,7); + INSERT INTO outer VALUES(1,2,3,4,5,6,7); + INSERT INTO right VALUES(1,2,3,4,5,6,7); +} +do_execsql_test func8-110 { + SELECT cross(cross,full,inner,left,natural,outer,right) FROM cross; +} cross-1-2-3-4-5-6-7 +do_execsql_test func8-120 { + SELECT full(cross,full,inner,left,natural,outer,right) FROM full; +} full-1-2-3-4-5-6-7 +do_execsql_test func8-130 { + SELECT inner(cross,full,inner,left,natural,outer,right) FROM inner; +} inner-1-2-3-4-5-6-7 +do_execsql_test func8-140 { + SELECT left(cross,full,inner,left,natural,outer,right) FROM left; +} left-1-2-3-4-5-6-7 +do_execsql_test func8-150 { + SELECT natural(cross,full,inner,left,natural,outer,right) FROM natural; +} natural-1-2-3-4-5-6-7 +do_execsql_test func8-160 { + SELECT outer(cross,full,inner,left,natural,outer,right) FROM outer; +} outer-1-2-3-4-5-6-7 +do_execsql_test func8-170 { + SELECT right(cross,full,inner,left,natural,outer,right) FROM right; +} right-1-2-3-4-5-6-7 + +finish_test diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index ee7749d969..9f509306b7 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1001,12 +1001,14 @@ static int recoverSqlCb(void *pCtx, const char *zSql){ */ static int recoverDatabase(sqlite3 *db){ int rc; /* Return code from this routine */ + const char *zRecoveryDb = ""; /* Name of "recovery" database */ const char *zLAF = "lost_and_found"; /* Name of "lost_and_found" table */ int bFreelist = 1; /* True to scan the freelist */ int bRowids = 1; /* True to restore ROWID values */ - sqlite3_recover *p; /* The recovery object */ + sqlite3_recover *p = 0; /* The recovery object */ p = sqlite3_recover_init_sql(db, "main", recoverSqlCb, 0); + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); @@ -1038,7 +1040,7 @@ static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){ printf("RUNNING-SQL: [%s]\n", zSql); fflush(stdout); } - (*pBtsFlags) &= ~BTS_BADPRAGMA; + (*pBtsFlags) &= BTS_BADPRAGMA; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc==SQLITE_OK ){ int nRow = 0; @@ -1181,6 +1183,7 @@ int runCombinedDbSqlInput( sqlite3_free(aDb); return 1; } + sqlite3_db_config(cx.db, SQLITE_DBCONFIG_STMT_SCANSTATUS, 1, 0); if( bVdbeDebug ){ sqlite3_exec(cx.db, "PRAGMA vdbe_debug=ON", 0, 0, 0); } diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 883f8cdfc9..5d473f19f3 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -103,7 +103,7 @@ int fuzz_invariant( } if( eVerbosity>=2 ){ char *zSql = sqlite3_expanded_sql(pTestStmt); - printf("invariant-sql #%d:\n%s\n", iCnt, zSql); + printf("invariant-sql row=%d #%d:\n%s\n", iRow, iCnt, zSql); sqlite3_free(zSql); } while( (rc = sqlite3_step(pTestStmt))==SQLITE_ROW ){ @@ -115,6 +115,8 @@ int fuzz_invariant( if( rc==SQLITE_DONE ){ /* No matching output row found */ sqlite3_stmt *pCk = 0; + int iOrigRSO; + /* This is not a fault if the database file is corrupt, because anything ** can happen with a corrupt database file */ @@ -124,6 +126,12 @@ int fuzz_invariant( sqlite3_finalize(pTestStmt); return rc; } + if( eVerbosity>=2 ){ + char *zSql = sqlite3_expanded_sql(pCk); + printf("invariant-validity-check #1:\n%s\n", zSql); + sqlite3_free(zSql); + } + rc = sqlite3_step(pCk); if( rc!=SQLITE_ROW || sqlite3_column_text(pCk, 0)==0 @@ -136,28 +144,29 @@ int fuzz_invariant( } sqlite3_finalize(pCk); - if( sqlite3_strlike("%group%by%",sqlite3_sql(pStmt),0)==0 ){ - /* - ** If there is a GROUP BY clause, it might not cover every term in the - ** output. And then non-covered terms can take on a value from any - ** row in the result set. This can cause differing answers. - */ - goto not_a_fault; + /* + ** If inverting the scan order also results in a miss, assume that the + ** query is ambiguous and do not report a fault. + */ + sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, -1, &iOrigRSO); + sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, !iOrigRSO, 0); + sqlite3_prepare_v2(db, sqlite3_sql(pStmt), -1, &pCk, 0); + sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, iOrigRSO, 0); + if( eVerbosity>=2 ){ + char *zSql = sqlite3_expanded_sql(pCk); + printf("invariant-validity-check #2:\n%s\n", zSql); + sqlite3_free(zSql); } - - if( sqlite3_strlike("%limit%)%order%by%", sqlite3_sql(pTestStmt),0)==0 ){ - /* crash-89bd6a6f8c6166e9a4c5f47b3e70b225f69b76c6 - ** Original statement is: - ** - ** SELECT a,b,c* FROM t1 LIMIT 1%5<4 - ** - ** When running: - ** - ** SELECT * FROM (...) ORDER BY 1 - ** - ** A different subset of the rows come out - */ - goto not_a_fault; + while( (rc = sqlite3_step(pCk))==SQLITE_ROW ){ + for(i=0; i=nCol ) break; + } + sqlite3_finalize(pCk); + if( rc==SQLITE_DONE ){ + sqlite3_finalize(pTestStmt); + return SQLITE_DONE; } /* The original sameValue() comparison assumed a collating sequence @@ -169,6 +178,12 @@ int fuzz_invariant( "SELECT ?1=?2 OR ?1=?2 COLLATE nocase OR ?1=?2 COLLATE rtrim", -1, &pCk, 0); if( rc==SQLITE_OK ){ + if( eVerbosity>=2 ){ + char *zSql = sqlite3_expanded_sql(pCk); + printf("invariant-validity-check #3:\n%s\n", zSql); + sqlite3_free(zSql); + } + sqlite3_reset(pTestStmt); while( (rc = sqlite3_step(pTestStmt))==SQLITE_ROW ){ for(i=0; i=2 ){ + char *zSql = sqlite3_expanded_sql(pCk); + printf("invariant-validity-check #4:\n%s\n", zSql); + sqlite3_free(zSql); + } sqlite3_bind_pointer(pCk, 1, pStmt, "stmt-pointer", 0); rc = sqlite3_step(pCk); } diff --git a/test/gencol1.test b/test/gencol1.test index ee0ebc53f5..f3fbb0dfba 100644 --- a/test/gencol1.test +++ b/test/gencol1.test @@ -615,4 +615,58 @@ do_execsql_test gencol1-22.1 { AND (y.a=2 OR (x.b LIKE '2*' AND y.a=x.b)); } {2 2 2 2} + +# 2023-03-02 dbsqlfuzz 65f5eb57f8859344d5f1f33e08c77ee12960ed83 +# +set typelist {ANY INT REAL BLOB TEXT {}} +set cnt 0 +foreach t1 $typelist { + foreach t2 $typelist { + incr cnt + db eval " + DROP TABLE IF EXISTS t1; + CREATE TABLE t1( + x $t1, + a $t2 AS (x) VIRTUAL, + b BLOB AS (x) VIRTUAL + ); + CREATE INDEX x2 ON t1(a); + INSERT INTO t1(x) VALUES(NULL),('1'),(2),(3.5),('xyz'); + " + set x1 [lsort [db eval {SELECT typeof(b) FROM t1}]] + do_test gencol1-23.1.$cnt { + lsort [db eval {SELECT typeof(b) FROM t1 INDEXED BY x2}] + } $x1 + } +} +do_execsql_test gencol1-23.2 { + DROP TABLE t1; + CREATE TABLE t1( + x, + a INT AS (x) VIRTUAL, + b BLOB AS (x) VIRTUAL + ); + CREATE INDEX x2 ON t1(a); + INSERT INTO t1(x) VALUES(NULL),('1'),('xyz'),(2),(3.5); + SELECT quote(a) FROM t1 INDEXED BY x2; +} {NULL 1 2 3.5 'xyz'} +do_execsql_test gencol1-23.3 { + EXPLAIN SELECT a FROM t1 INDEXED BY x2; +} {~/Column 0/} +# ^^^^^^^^---- verfies that x2 acts like a covering index +do_execsql_test gencol1-23.4 { + EXPLAIN SELECT b FROM t1 INDEXED BY x2; +} {/Column 0/} +# ^^^^^^^^^^--- Must reference the original table in this case because +# of the different datatype on column b. + +# 2023-03-07 https://sqlite.org/forum/forumpost/b312e075b5 +# +do_execsql_test gencol1-23.5 { + CREATE TABLE v0(c1 INT, c2 AS (RAISE(IGNORE))); +} +do_catchsql_test gencol1-23.6 { + SELECT * FROM v0; +} {1 {RAISE() may only be used within a trigger-program}} + finish_test diff --git a/test/in.test b/test/in.test index 716c17f593..601c7e3b4d 100644 --- a/test/in.test +++ b/test/in.test @@ -829,7 +829,35 @@ do_execsql_test in-22.4 { SELECT * FROM t1 WHERE x IN ((((((SELECT a FROM t2)))))); } {2 200 4 400 6 600} - - +# 2023-04-04 https://sqlite.org/forum/forumpost/dc16ec63d3 +# Faulty assert() statement in the IN optimization. +# +do_execsql_test in-23.0 { + DROP TABLE IF EXISTS t4; + CREATE TABLE t4(a TEXT, b INT); + INSERT INTO t4(a,b) VALUES('abc',0),('ABC',1),('def',2); + CREATE INDEX t4x ON t4(a, +a COLLATE NOCASE); + SELECT a0.a, group_concat(a1.a) AS b + FROM t4 AS a0 JOIN t4 AS a1 + GROUP BY a0.a + HAVING (SELECT sum( (a1.a == +a0.a COLLATE NOCASE) IN (SELECT b FROM t4))); +} {ABC abc,ABC,def abc abc,ABC,def def abc,ABC,def} +do_execsql_test in-23.0-b { + SELECT a0.a, group_concat(a1.a) AS b + FROM t4 AS a0 JOIN t4 AS a1 + GROUP BY a0.a + HAVING (SELECT sum( (a1.a GLOB +a0.a COLLATE NOCASE) IN (SELECT b FROM t4))); +} {ABC abc,ABC,def abc abc,ABC,def def abc,ABC,def} +# +# Follow-up forum/forumpost/0713a16a44 +# +do_execsql_test in-23.1 { + CREATE VIEW t5 AS + SELECT 1 AS b + WHERE (SELECT count(0=NOT+a COLLATE NOCASE IN (SELECT 0)) + FROM t4 + GROUP BY a); + SELECT * FROM t5; +} 1 finish_test diff --git a/test/indexexpr1.test b/test/indexexpr1.test index 7d94806cdd..51ef73bbf5 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -591,6 +591,29 @@ do_execsql_test indexexpr1-2140 { SELECT b FROM t1; } 400 - +# 2023-04-18 Forum post https://sqlite.org/forum/forumpost/f34e32d120 from +# Alexis King. +# +# This problem originates at check-in b9190d3da70c4171 (2022-11-25). +# A similar problem arose on 2023-03-04 at +# https://sqlite.org/forum/forumpost/a68313d054 and was fixed at +# check-in e06973876993926f. See the test case tkt-99378-400. +# +reset_db +do_execsql_test indexexpr1-2200 { + CREATE TABLE t1(id INTEGER PRIMARY KEY, tag INT); + INSERT INTO t1 VALUES (0, 7), (1, 8); + CREATE TABLE t2(type INT, t1_id INT, value INT); + INSERT INTO t2 VALUES (0, 0, 100), (0, 1, 101); + CREATE INDEX t1x ON t1(-tag); + SELECT u.tag, v.max_value + FROM (SELECT tag FROM t1 GROUP BY -tag) u + JOIN (SELECT t1.tag AS "tag", t2.type AS "type", + MAX(t2.value) AS "max_value" + FROM t1 + JOIN t2 ON t2.t1_id = t1.id + GROUP BY t2.type, t1.tag + ) v ON v.type = 0 AND v.tag = u.tag; +} {7 100 8 101} finish_test diff --git a/test/indexexpr2.test b/test/indexexpr2.test index 8c1171e038..4c21421e8e 100644 --- a/test/indexexpr2.test +++ b/test/indexexpr2.test @@ -372,4 +372,57 @@ foreach {tn expr} { " {1 1} } +# 2023-03-24 https://sqlite.org/forum/forumpost/79cf371080 +# +reset_db +do_execsql_test 9.0 { + CREATE TABLE t1(a INT, b INT); + CREATE INDEX t1x ON t1(a, abs(b)); + CREATE TABLE t2(c INT, d INT); + INSERT INTO t1(a,b) VALUES(4,4),(5,-5),(5,20),(6,6); + INSERT INTO t2(c,d) VALUES(100,1),(200,1),(300,2); + SELECT *, + (SELECT max(c+abs(b)) FROM t2 GROUP BY d ORDER BY d LIMIT 1) AS subq + FROM t1 WHERE a=5; +} {5 -5 205 5 20 220} + +# 2023-04-03 https://sqlite.org/forum/forumpost/44270909bb +# and https://sqlite.org/forum/forumpost/e45108732c which are the +# same problem, namely the failure to omit the EP_Collate property +# from an expression node when changing it from TK_COLLATE into +# TK_AGG_COLUMN because it resolves to an indexed expression. +# +reset_db +do_execsql_test 10.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + CREATE INDEX t1x ON t1 (b, +b COLLATE NOCASE); + INSERT INTO t1(a,b) VALUES(1,'abcde'); + SELECT * FROM t1 AS a0 + WHERE (SELECT count(a0.b=+a0.b COLLATE NOCASE IN (b)) FROM t1 GROUP BY 2.5) + ORDER BY a0.b; +} {1 abcde} +do_execsql_test 10.1 { + CREATE TABLE t2(a TEXT); + INSERT INTO t2 VALUES('alice'),('bob'),('cindy'),('david'); + CREATE INDEX t2x ON t2 (+a COLLATE NOCASE); + SELECT count(+a COLLATE NOCASE IN (SELECT 1)) AS x + FROM t2 + GROUP BY SUBSTR(0,0); +} 4 + +# 2023-04-03 https://sqlite.org/forum/forumpost/409ebc7368 +# When a generated column appears in both an outer and an inner loop +# (that is to say, the same table is used in both loops) and the +# generated column is indexed and it is used inside an aggregate function, +# make sure that the terms resolve to the correct aggregate. +# +do_execsql_test 11.0 { + CREATE TABLE t3 (a INT, b AS (-a)); + CREATE INDEX t3x ON t3(b, a); + INSERT INTO t3(a) VALUES(44); + SELECT * FROM t3 AS a0 + WHERE (SELECT sum(-a0.a=b) FROM t3 GROUP BY b) + GROUP BY b; +} {44 -44} + finish_test diff --git a/test/join.test b/test/join.test index f0ac0be8b1..44bfb3bef2 100644 --- a/test/join.test +++ b/test/join.test @@ -1038,18 +1038,56 @@ do_execsql_test join-22.10 { } {11} # 2019-12-22 ticket 7929c1efb2d67e98 +# Verification of testtag-20230227a +# +# 2023-02-27 https://sqlite.org/forum/forumpost/422e635f3beafbf6 +# Verification of testtag-20230227a, testtag-20230227b, and testtag-20230227c # reset_db ifcapable vtab { -do_execsql_test join-23.10 { - CREATE TABLE t0(c0); - INSERT INTO t0(c0) VALUES(123); - CREATE VIEW v0(c0) AS SELECT 0 GROUP BY 1; - SELECT t0.c0, v0.c0, vt0.name - FROM v0, t0 LEFT JOIN pragma_table_info('t0') AS vt0 - ON vt0.name LIKE 'c0' - WHERE v0.c0 == 0; -} {123 0 c0} + do_execsql_test join-23.10 { + CREATE TABLE t0(c0); + INSERT INTO t0(c0) VALUES(123); + CREATE VIEW v0(c0) AS SELECT 0 GROUP BY 1; + SELECT t0.c0, v0.c0, vt0.name + FROM v0, t0 LEFT JOIN pragma_table_info('t0') AS vt0 + ON vt0.name LIKE 'c0' + WHERE v0.c0 == 0; + } {123 0 c0} + do_execsql_test join-23.20 { + CREATE TABLE a(value TEXT); + INSERT INTO a(value) SELECT value FROM json_each('["a", "b", null]'); + CREATE TABLE b(value TEXT); + INSERT INTO b(value) SELECT value FROM json_each('["a", "c", null]'); + SELECT a.value, b.value FROM a RIGHT JOIN b ON a.value = b.value; + } {a a {} c {} {}} + do_execsql_test join-23.21 { + SELECT a.value, b.value FROM b LEFT JOIN a ON a.value = b.value; + } {a a {} c {} {}} + do_execsql_test join-23.22 { + SELECT a.value, b.value + FROM json_each('["a", "c", null]') AS b + LEFT JOIN + json_each('["a", "b", null]') AS a ON a.value = b.value; + } {a a {} c {} {}} + do_execsql_test join-23.23 { + SELECT a.value, b.value + FROM json_each('["a", "b", null]') AS a + RIGHT JOIN + json_each('["a", "c", null]') AS b ON a.value = b.value; + } {a a {} c {} {}} + do_execsql_test join-23.24 { + SELECT a.value, b.value + FROM json_each('["a", "b", null]') AS a + RIGHT JOIN + b ON a.value = b.value; + } {a a {} c {} {}} + do_execsql_test join-23.25 { + SELECT a.value, b.value + FROM a + RIGHT JOIN + json_each('["a", "c", null]') AS b ON a.value = b.value; + } {a a {} c {} {}} } #------------------------------------------------------------------------- @@ -1207,4 +1245,22 @@ do_eqp_test join-28.2 { # |--SCAN t4 # `--SEARCH t3 USING AUTOMATIC COVERING INDEX (a=?) + +# 2023-05-01 https://sqlite.org/forum/forumpost/96cd4a7e9e +# +reset_db +db null NULL +do_execsql_test join-29.1 { + CREATE TABLE t0(a INT); INSERT INTO t0(a) VALUES (1); + CREATE TABLE t1(b INT); INSERT INTO t1(b) VALUES (2); + CREATE VIEW v2(c) AS SELECT 3 FROM t1; + SELECT * FROM t1 JOIN v2 ON 0 FULL OUTER JOIN t0 ON true; +} {NULL NULL 1} +do_execsql_test join-29.2 { + SELECT * FROM t1 JOIN v2 ON 1=0 FULL OUTER JOIN t0 ON true; +} {NULL NULL 1} +do_execsql_test join-29.3 { + SELECT * FROM t1 JOIN v2 ON false FULL OUTER JOIN t0 ON true; +} {NULL NULL 1} + finish_test diff --git a/test/join2.test b/test/join2.test index e549c49343..15e76f965d 100644 --- a/test/join2.test +++ b/test/join2.test @@ -354,5 +354,78 @@ do_execsql_test 9.11 { SELECT ccc, ccc IS NULL AS ddd FROM t1 LEFT JOIN v2; } {{} 1} +# 2023-03-01 https://sqlite.org/forum/forumpost/26387ea7ef +# When flattening a VIEW which is the RHS of a LEFT JOIN, always put +# an TK_IF_NULL_ROW operator on all accesses, even TK_COLUMN nodes, since +# the TK_COLUMN might reference an outer subquery. +# +reset_db +db null NULL +do_execsql_test 10.1 { + CREATE TABLE t1 (x INTEGER); + INSERT INTO t1 VALUES(1); -- Some true value + CREATE TABLE t2 (z TEXT); + INSERT INTO t2 VALUES('some value'); + CREATE TABLE t3(w TEXT); + INSERT INTO t3 VALUES('some other value'); +} +do_execsql_test 10.2 { + SELECT ( + SELECT 1 FROM t2 LEFT JOIN (SELECT x AS v FROM t3) ON 500=v WHERE (v OR FALSE) + ) FROM t1; +} NULL +do_execsql_test 10.3 { + SELECT ( + SELECT 1 FROM t2 LEFT JOIN (SELECT x AS v FROM t3) ON 500=v WHERE (v) + ) FROM t1; +} NULL +optimization_control db all 0 +do_execsql_test 10.4 { + SELECT ( + SELECT 1 FROM t2 LEFT JOIN (SELECT x AS v FROM t3) ON 500=v WHERE (v OR FALSE) + ) FROM t1; +} NULL + +# 2023-03-02 https://sqlite.org/forum/forumpost/402f05296d +# +# The TK_IF_NULL_ROW expression node must ensure that it does not overwrite +# the result register of an OP_Once subroutine. +# +optimization_control db all 1 +do_execsql_test 11.1 { + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; + CREATE TABLE t1(x TEXT, y INTEGER); + INSERT INTO t1(x,y) VALUES(NULL,-2),(NULL,1),('0',2); + CREATE TABLE t2(z INTEGER); + INSERT INTO t2(z) VALUES(2),(-2); + CREATE VIEW t3 AS SELECT z, (SELECT count(*) FROM t1) AS w FROM t2; + SELECT * FROM t1 LEFT JOIN t3 ON y=z; +} {NULL -2 -2 3 NULL 1 NULL NULL 0 2 2 3} + +# 2023-03-11 https://sqlite.org/forum/forumpost/b405033490fa56d9 +# The fix that test 11.1 above checks also caused a performance regression. +# This test case verifies that the performance regression has been resolved. +# +do_execsql_test 12.1 { + DROP TABLE t1; + DROP TABLE t2; + DROP VIEW t3; + CREATE TABLE t1(a INTEGER PRIMARY KEY); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<100) + INSERT INTO t1(a) SELECT n FROM c; + CREATE VIEW t2(b) AS SELECT a FROM t1; +} +do_vmstep_test 12.2 { + SELECT * FROM t1 LEFT JOIN t2 ON a=b LIMIT 10 OFFSET 98; +} 2000 {99 99 100 100} +do_eqp_test 12.3 { + SELECT * FROM t1 LEFT JOIN t2 ON a=b LIMIT 10 OFFSET 98; +} { + QUERY PLAN + |--SCAN t1 + `--SEARCH t1 USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN +} finish_test diff --git a/test/join8.test b/test/join8.test index 1140c104f0..fc50df32ff 100644 --- a/test/join8.test +++ b/test/join8.test @@ -159,6 +159,7 @@ do_catchsql_test join8-5000 { } {0 {- -}} # 2022-04-29 dbsqlfuzz 19f1102a70cf966ab249de56d944fc20dbebcfcf +# Verification of testtag-20230227b and testtag-20230227c # reset_db do_execsql_test join8-6000 { @@ -181,6 +182,16 @@ do_execsql_test join8-6020 { SELECT value, t1.* FROM json_each('17') NATURAL RIGHT JOIN t1 WHERE (a,b) IN (SELECT rowid, b FROM t1); } {17 1 2} +do_execsql_test join8-6021 { + SELECT value, t1.* FROM json_each('null') NATURAL RIGHT JOIN t1 + WHERE (a,b) IN (SELECT rowid, b FROM t1); +} {{} 1 2} +do_execsql_test join8-6022 { + CREATE TABLE a(key TEXT); + INSERT INTO a(key) VALUES('a'),('b'); + SELECT quote(a.key), b.value + FROM a RIGHT JOIN json_each('["a","c"]') AS b ON a.key=b.value; +} {'a' a NULL c} # Bloom filter usage by RIGHT and FULL JOIN # diff --git a/test/joinH.test b/test/joinH.test index 1d5f66afa2..78d1556293 100644 --- a/test/joinH.test +++ b/test/joinH.test @@ -89,5 +89,35 @@ do_execsql_test 4.4 { SELECT (d IS NULL) FROM t1 RIGHT JOIN t2 ON (j=33); } {1} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.0 { + CREATE TABLE t0(w); + CREATE TABLE t1(x); + CREATE TABLE t2(y); + CREATE TABLE t3(z); + INSERT INTO t3 VALUES('t3val'); +} + +do_execsql_test 5.1 { + SELECT * FROM t1 INNER JOIN t2 ON (0) RIGHT OUTER JOIN t3; +} {{} {} t3val} + +do_execsql_test 5.2 { + SELECT * FROM t1 INNER JOIN t2 ON (0) FULL OUTER JOIN t3; +} {{} {} t3val} + +do_execsql_test 5.3 { + SELECT * FROM t3 LEFT JOIN t2 ON (0); +} {t3val {}} + +do_execsql_test 5.4 { + SELECT * FROM t0 RIGHT JOIN t1 INNER JOIN t2 ON (0) RIGHT JOIN t3 +} {{} {} {} t3val} + +do_execsql_test 5.5 { + SELECT * FROM t0 RIGHT JOIN t1 INNER JOIN t2 ON (0) +} {} finish_test diff --git a/test/json/README.md b/test/json/README.md new file mode 100644 index 0000000000..6a16114925 --- /dev/null +++ b/test/json/README.md @@ -0,0 +1,27 @@ +The files in this subdirectory are used to help measure the performance +of the SQLite JSON parser. + +# 1.0 Prerequisites + + 1. Valgrind + + 2. Fossil + +# 2.0 Setup + + 1. Run: "`tclsh json-generator.tcl | sqlite3 json100mb.db`" to create + the 100 megabyte test database. Do this so that the "json100mb.db" + file lands in the directory from which you will run tests, not in + the test/json subdirectory of the source tree. + + 2. Build the baseline sqlite3.c file. ("`make sqlite3.c`") + + 3. Run "`sh json-speed-check-1.sh trunk`". This creates the baseline + profile in "jout-trunk.txt". + +# 3.0 Testing + + 1. Build the sqlite3.c to be tested. + + 2. Run "`sh json-speed-check-1.sh x1`". The profile output will appear + in jout-x1.txt. Substitute any label you want in place of "x1". diff --git a/test/json/json-generator.tcl b/test/json/json-generator.tcl new file mode 100644 index 0000000000..d499bc7300 --- /dev/null +++ b/test/json/json-generator.tcl @@ -0,0 +1,401 @@ +#!/usr/bin/tclsh +# +# Generate SQL that will populate an SQLite database with about 100 megabytes +# of pseudo-random JSON text. +# +# tclsh json-generator.tcl | sqlite3 json110mb.db +# +# srand() is used to initialize the random seed so that the same JSON +# is generated for every run. +# +expr srand(12345678) +set wordlist { + ability able abroad access account act + action active actor add address adept + adroit advance advice affect age ageless + agency agent agile agree air airfare + airline airport alert almond alpha always + amend amount amplify analyst anchor angel + angelic angle ankle annual answer antique + anybody anyhow appeal apple apricot apt + area argon arm army arrival arsenic + art artful article arugula aside ask + aspect assist assume atom atone attempt + author autumn average avocado award awl + azure back bacon bag bagel bake + baker balance ball balloon bamboo banana + band banjo bank barium base basil + basin basis basket bass bat bath + battery beach beak bean bear bearcub + beauty beef beet beige being bell + belly belt bench bend benefit best + beta better beyond bicycle bid big + bike bill bird biscuit bismuth bisque + bit black blank blest blind bliss + block bloom blue board boat body + bokchoy bone bonus book bookish boot + border boron boss bossy bottle bottom + bow bowl bowtie box brain brainy + branch brave bravely bread break breath + breezy brick bridge brie brief briefly + bright broad broil bromine bronze brother + brow brown brush buddy budget buffalo + bug bugle bull bunch burger burly + burrito bus busy butter button buy + buyer byte cab cabbage cabinet cable + cadet cadmium caesium cake calcium caliper + call caller calm calmly camera camp + can canary cancel candle candy cap + capable caper capital captain car carbon + card care career careful carp carpet + carrot carry case cash cassava casual + cat catch catfish catsear catsup cause + cave celery cell century chain chair + chalk chance change channel chapter chard + charge charity chart check cheddar cheery + cheese chicken chicory chiffon child chin + chip chives choice chowder chum church + circle city claim clam class classic + classy clay clean cleaner clear clearly + clerk click client climate clock clorine + closet clothes cloud clown club clue + cluster coach coast coat cobbler cobolt + cod code coffee colby cold collar + college comb combine comet comfort command + comment common company complex concept concern + concert conduit consist contact contest context + control convert cook cookie copilot copper + copy coral cordial corn corner corny + correct cost count counter country county + couple courage course court cover cow + cowbird crab crack craft crash crazy + cream credit creek cress crevice crew + crimson croaker crop cross crowd cube + cuckoo cuisine culture cup current curve + cut cyan cycle dagger daily dance + dare darter data date day daylily + deal dear dearly debate debit decade + decimal deep deft deftly degree delay + deluxe deposit depth design desk detail + device dew diamond diet dig dill + dinner dip direct dirt dish disk + display diver divide divine doctor dodger + donut door dot double dough draft + drag dragon drama draw drawer drawing + dream drill drink drive driver drop + drum dry dryer drywall duck due + dump dusk dust duty dye eagle + ear earring earth ease east easy + eat economy edge editor eel effect + effort egg eight elbow elegant element + elf elk email emerald employ end + endive endless energy engine enjoy enter + entry equal equip error escape essay + eternal evening event exam example excuse + exit expert extent extreme eye face + fact factor factual fail failure fair + fajita fall family fan fang farm + farmer fat fault feature feed feel + feeling fench fennel festive few fiber + field fig figure file fill film + filter final finance finding finger finish + fire fish fishing fit fitting five + fix flier flight floor floral florine + flour flow flower fly flying focus + fold folding food foot force forest + forever forgive form formal format fortune + forum frame free freedom freely fresh + friend frog front fruit fuchsia fuel + fun funny future gain galaxy gallium + game gamma gap garage garden garlic + gas gate gather gauge gear gem + gene general gentle gently gherkin ghost + gift give glad glass gleeful glossy + glove glue goal goat goby gold + goldeye golf good gouda goulash gourd + grab grace grade gram grand grape + grapes grass gravy gray great green + grits grocery ground group grouper grout + growth guard guave guess guest guide + guitar gumbo guppy habit hacksaw haddock + hafnium hagfish hair half halibut hall + hammer hand handle handy hanger happy + hat havarti hay haybale head health + healthy hearing heart hearty heat heavy + heel height helium hello help helpful + herald herring hide high highly highway + hill hip hipster hire history hit + hoki hold hole holiday holly home + honest honey hook hope hopeful horizon + horn horse host hotel hour house + housing human humane humor hunt hurry + ice icecube icefish icy idea ideal + image impact impress inch income indigo + initial inkpen insect inside intense invite + iodine iridium iron island issue item + ivory jacket jargon javelin jello jelly + jewel job jocund join joint joke + jovial joy joyful joyous judge juice + jump junior jury just justice kale + keel keep kelp ketchup key keyhole + keyway khaki kick kid kidney kiloohm + kind kindly king kitchen kite kiwi + knee knife krill krypton kumquat lab + lace lack ladder lake lamp lamprey + land laser laugh law lawn lawyer + layer lead leader leading leaf leafy + league leather leave lecture leek leg + lemon length lentil lesson let letter + lettuce level library life lift light + lily lime limit line linen link + lip list listen lithium lively living + lizard load loan lobster local lock + log long longfin look lotus love + lovely loving low lucid luck luffa + lunch lung machine magenta magnet mail + main major make mall manager mango + manner many map march market maroon + martian master match math matter maximum + maybe meal meaning meat media medium + meet meeting melody melon member memory + mention menu mercury merry mess message + messy metal meter method micron middle + might mile milk mind mine minimum + minnow minor mint minute mirror miss + mission misty mix mixer mixture mobile + mode model moment monitor monk month + moon moray morning most motor mouse + mouth move mover movie much mud + mudfish muffin mullet munster muon muscle + music mustard nail name nation native + natural nature navy neat neatly nebula + neck needle neon nerve net network + neutron news nibble nice nickel night + niobium nobody noise noodle normal north + nose note nothing notice nova novel + number nurse nursery oar object offer + office officer oil okay okra old + olive one onion open opening opinion + option orange orbit orchid order oregano + other ounce outcome outside oven owner + oxygen oyster pace pack package page + pager paint pair pale pan pancake + papaya paper pardon parent park parking + parsley parsnip part partner party pass + passage past pasta path patient pattern + pause pay pea peace peach peacock + peahen peak peanut pear pearl pen + penalty pencil pension people pepper perch + perfect period permit person phase phone + photo phrase physics piano pick picture + pie piece pigeon pike pilot pin + pink pinkie pious pipe pitch pizza + place plan plane planet plant planter + plastic plate play player playful plenty + pliers plum pod poem poet poetry + point police policy pollock pony pool + pop popover poptart pork port portal + post pot potato pound powder power + present press price pride primary print + prior private prize problem process produce + product profile profit program project promise + prompt proof proper protein proton public + puff puffer pull pumpkin pup pupfish + pure purple purpose push put quality + quark quarter quiet quill quit quote + rabbit raccoon race radiant radio radish + radium radon rain rainbow raise ramp + ranch range rasp rate ratio ray + razor reach read reading real reality + reason recipe record recover red redeem + reed reef refuse region regret regular + relaxed release relief relish remote remove + rent repair repeat reply report request + reserve resist resolve resort rest result + return reveal review reward ribbon rice + rich ride ridge right ring rise + risk river rivet road roast rock + rocket role roll roof room rope + rose rough roughy round row royal + rub ruby rudder ruin rule run + runner rush rust sacred saddle safe + safety sail salad salami sale salmon + salt sample sand sander sandy sauce + save saving saw scale scampi scene + scheme school score screen script sea + search season seat second secret sector + seemly self sell senate senior sense + series serve set shake shape share + shark shell shift shine shiny ship + shock shoe shoot shop shovel show + side sign signal silk silly silver + simple sing singer single sink site + size skill skin sky slate sleep + sleepy slice slide slip smart smell + smelt smile smoke smooth snap snipe + snow snowy sock socket sodium soft + softly soil sole solid song sorrel + sort soul sound soup source south + space spare speech speed spell spend + sphere spice spider spirit spite split + spoon sport spot spray spread spring + squab square squash stable staff stage + stand staple star start state status + stay steak steel step stern stew + stick still stock stone stop store + storm story strain street stress strike + string stroke strong studio study stuff + style sugar suit sulfur summer sun + sunny sunset super superb surf survey + sweet swim swing switch symbol system + table tackle tail tale talk tan + tank tap tape target task taste + tau tea teach teal team tear + tell ten tender tennis tent term + test tetra text thanks theme theory + thing think thread throat thumb ticket + tidy tie tiger till time timely + tin tip title toast today toe + tomato tone tongue tool tooth top + topic total touch tough tour towel + tower town track trade train trash + travel tray treat tree trick trip + trout trowel truck trupet trust truth + try tube tuna tune turf turkey + turn turnip tutor tux tweet twist + two type union unique unit upbeat + upper use useful user usual valley + value van vase vast veil vein + velvet verse very vessel vest video + view violet visit visual vivid voice + volume vowel voyage waffle wait wake + walk wall warm warmth wasabi wash + watch water wave wax way wealth + wear web wedge week weekly weight + west whale what wheat wheel when + where while who whole why will + win wind window wing winner winter + wire wish witty wolf wonder wood + wool woolly word work worker world + worry worth worthy wrap wrench wrist + writer xenon yak yam yard yarrow + year yearly yellow yew yogurt young + youth zebra zephyr zinc zone zoo +} +set nwordlist [llength $wordlist] + +proc random_char {} { + return [string index \ + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" \ + [expr {int(rand()*52)}]] +} +proc random_label {} { + set label [random_char] + while {rand()>0.8} { + append label [random_char] + } + if {rand()>0.9} {append label -} + append label [format %d [expr {int(rand()*100)}]] + return $label +} +proc random_numeric {} { + set n [expr {(rand()*2-1.0)*1e6}] + switch [expr {int(rand()*6)}] { + 0 {set format %.3f} + 1 {set format %.6E} + 2 {set format %.4e} + default {set format %g} + } + return [format $format $n] +} + + +proc random_json {limit indent} { + global nwordlist wordlist + set res {} + if {$indent==0 || ($limit>0 && rand()>0.5)} { + incr limit -1 + incr indent 2 + set n [expr {int(rand()*5)+1}] + if {$n==5} {incr n [expr {int(rand()*10)}]} + if {rand()>0.5} { + set res \173\n + for {set i 0} {$i<$n} {incr i} { + append res [string repeat { } $indent] + if {rand()>0.8} { + if {rand()>0.5} { + set sep ":\n [string repeat { } $indent]" + } else { + set sep " : " + } + } else { + set sep : + } + append res \"[random_label]\"$sep[random_json $limit $indent] + if {$i<$n-1} {append res ,} + append res \n + } + incr indent -2 + append res [string repeat { } $indent] + append res \175 + return $res + } else { + set res \[\n + for {set i 0} {$i<$n} {incr i} { + append res [string repeat { } $indent] + append res [random_json $limit $indent] + if {$i<$n-1} {append res ,} + append res \n + } + incr indent -2 + append res [string repeat { } $indent] + append res \] + return $res + } + } elseif {rand()>0.9} { + if {rand()>0.7} {return "true"} + if {rand()>0.5} {return "false"} + return "null" + } elseif {rand()>0.5} { + return [random_numeric] + } else { + set res \" + set n [expr {int(rand()*4)+1}] + if {$n>=4} {set n [expr {$n+int(rand()*6)}]} + for {set i 0} {$i<$n} {incr i} { + if {rand()<0.05} { + set w [random_numeric] + } else { + set k [expr {int(rand()*$nwordlist)}] + set w [lindex $wordlist $k] + } + if {rand()<0.07} { + set w \\\"$w\\\" + } + if {$i<$n-1} { + switch [expr {int(rand()*9)}] { + 0 {set sp {, }} + 1 {set sp "\\n "} + 2 {set sp "-"} + default {set sp { }} + } + append res $w$sp + } else { + append res $w + if {rand()<0.2} {append res .} + } + } + return $res\" + } +} + +puts "CREATE TABLE IF NOT EXISTS data1(x JSON);" +puts "BEGIN;" +set sz 0 +for {set i 0} {$sz<100000000} {incr i} { + set j [random_json 7 0] + incr sz [string length $j] + puts "INSERT INTO data1(x) VALUES('$j');" +} +puts "COMMIT;" +puts "SELECT sum(length(x)) FROM data1;" diff --git a/test/json/json-q1.txt b/test/json/json-q1.txt new file mode 100644 index 0000000000..0395f0c061 --- /dev/null +++ b/test/json/json-q1.txt @@ -0,0 +1,4 @@ +.mode qbox +.timer on +.param set $label 'q87' +SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL; diff --git a/test/json/json-speed-check.sh b/test/json/json-speed-check.sh new file mode 100755 index 0000000000..f7e7f1be58 --- /dev/null +++ b/test/json/json-speed-check.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# This is a template for a script used for day-to-day size and +# performance monitoring of SQLite. Typical usage: +# +# sh speed-check.sh trunk # Baseline measurement of trunk +# sh speed-check.sh x1 # Measure some experimental change +# fossil xdiff --tk jout-trunk.txt jout-x1.txt # View chanages +# +# There are multiple output files, all with a base name given by +# the first argument: +# +# summary-$BASE.txt # Copy of standard output +# jout-$BASE.txt # cachegrind output +# explain-$BASE.txt # EXPLAIN listings (only with --explain) +# +if test "$1" = "" +then + echo "Usage: $0 OUTPUTFILE [OPTIONS]" + exit +fi +NAME=$1 +shift +#CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5" +CC_OPTS="-DSQLITE_ENABLE_MEMSYS5" +CC=gcc +LEAN_OPTS="-DSQLITE_THREADSAFE=0" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_MEMSTATUS=0" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOBS" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_MAX_EXPR_DEPTH=0" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DECLTYPE" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DEPRECATED" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA" +BASELINE="trunk" +doExplain=0 +doCachegrind=1 +doVdbeProfile=0 +doWal=1 +doDiff=1 +while test "$1" != ""; do + case $1 in + --nodiff) + doDiff=0 + ;; + --lean) + CC_OPTS="$CC_OPTS $LEAN_OPTS" + ;; + --clang) + CC=clang + ;; + --gcc7) + CC=gcc-7 + ;; + -*) + CC_OPTS="$CC_OPTS $1" + ;; + *) + BASELINE=$1 + ;; + esac + shift +done +echo "NAME = $NAME" | tee summary-$NAME.txt +echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt +rm -f cachegrind.out.* jsonshell +$CC -g -Os -Wall -I. $CC_OPTS ./shell.c ./sqlite3.c -o jsonshell -ldl -lpthread +ls -l jsonshell | tee -a summary-$NAME.txt +home=`echo $0 | sed -e 's,/[^/]*$,,'` +echo ./jsonshell json100mb.db "<$home/json-q1.txt" +valgrind --tool=cachegrind ./jsonshell json100mb.db <$home/json-q1.txt \ + 2>&1 | tee -a summary-$NAME.txt +cg_anno.tcl cachegrind.out.* >jout-$NAME.txt +echo '*****************************************************' >>jout-$NAME.txt +sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>jout-$NAME.txt +if test "$NAME" != "$BASELINE"; then + fossil xdiff --tk -c 20 jout-$BASELINE.txt jout-$NAME.txt +fi diff --git a/test/json101.test b/test/json101.test index 135c988e97..e17c8cd6a2 100644 --- a/test/json101.test +++ b/test/json101.test @@ -310,12 +310,33 @@ do_execsql_test json-6.1 { SELECT json_valid('{"a":55,"b":72,}'); } {0} do_execsql_test json-6.2 { + SELECT json_error_position('{"a":55,"b":72,}'); +} {0} +do_execsql_test json-6.3 { + SELECT json_valid(json('{"a":55,"b":72,}')); +} {1} +do_execsql_test json-6.4 { + SELECT json_valid('{"a":55,"b":72 , }'); +} {0} +do_execsql_test json-6.5 { + SELECT json_error_position('{"a":55,"b":72 , }'); +} {0} +do_execsql_test json-6.6 { + SELECT json_error_position('{"a":55,"b":72,,}'); +} {16} +do_execsql_test json-6.7 { SELECT json_valid('{"a":55,"b":72}'); } {1} -do_execsql_test json-6.3 { - SELECT json_valid('["a",55,"b",72,]'); +do_execsql_test json-6.8 { + SELECT json_error_position('["a",55,"b",72,]'); } {0} -do_execsql_test json-6.4 { +do_execsql_test json-6.9 { + SELECT json_error_position('["a",55,"b",72 , ]'); +} {0} +do_execsql_test json-6.10 { + SELECT json_error_position('["a",55,"b",72,,]'); +} {16} +do_execsql_test json-6.11 { SELECT json_valid('["a",55,"b",72]'); } {1} @@ -885,4 +906,109 @@ do_execsql_test json-19.3 { SELECT * FROM t1; } {} +# 2023-03-17 positive and negative infinities +# +do_execsql_test json-20.1 { + SELECT json_object('a',2e370,'b',-3e380); +} {{{"a":9.0e+999,"b":-9.0e+999}}} +do_execsql_test json-20.2 { + SELECT json_object('a',2e370,'b',-3e380)->>'a'; +} Inf +do_execsql_test json-20.3 { + SELECT json_object('a',2e370,'b',-3e380)->>'b'; +} {-Inf} + +# 2023-05-02 https://sqlite.org/forum/forumpost/06c6334412 +# JSON functions should normally return NULL when given +# a NULL value as the JSON input. +# +db null NULL +do_execsql_test json-21.1 { + SELECT json_valid(NULL); +} NULL +do_execsql_test json-21.2 { + SELECT json_error_position(NULL); +} NULL +do_execsql_test json-21.3 { + SELECT json(NULL); +} NULL +do_execsql_test json-21.4 { + SELECT json_array(NULL); +} {[null]} +do_execsql_test json-21.5 { + SELECT json_extract(NULL); +} NULL +do_execsql_test json-21.6 { + SELECT json_insert(NULL,'$',123); +} NULL +do_execsql_test json-21.7 { + SELECT NULL->0; +} NULL +do_execsql_test json-21.8 { + SELECT NULL->>0; +} NULL +do_execsql_test json-21.9 { + SELECT '{a:5}'->NULL; +} NULL +do_execsql_test json-21.10 { + SELECT '{a:5}'->>NULL; +} NULL +do_catchsql_test json-21.11 { + SELECT json_object(NULL,5); +} {1 {json_object() labels must be TEXT}} +do_execsql_test json-21.12 { + SELECT json_patch(NULL,'{a:5}'); +} NULL +do_execsql_test json-21.13 { + SELECT json_patch('{a:5}',NULL); +} NULL +do_execsql_test json-21.14 { + SELECT json_patch(NULL,NULL); +} NULL +do_execsql_test json-21.15 { + SELECT json_remove(NULL,'$'); +} NULL +do_execsql_test json-21.16 { + SELECT json_remove('{a:5,b:7}',NULL); +} NULL +do_execsql_test json-21.17 { + SELECT json_replace(NULL,'$.a',123); +} NULL +do_execsql_test json-21.18 { + SELECT json_replace('{a:5,b:7}',NULL,NULL); +} {{{"a":5,"b":7}}} +do_execsql_test json-21.19 { + SELECT json_set(NULL,'$.a',123); +} NULL +do_execsql_test json-21.20 { + SELECT json_set('{a:5,b:7}',NULL,NULL); +} {{{"a":5,"b":7}}} +do_execsql_test json-21.21 { + SELECT json_type(NULL); +} NULL +do_execsql_test json-21.22 { + SELECT json_type('{a:5,b:7}',NULL); +} NULL +do_execsql_test json-21.23 { + SELECT json_quote(NULL); +} null +do_execsql_test json-21.24 { + SELECT count(*) FROM json_each(NULL); +} 0 +do_execsql_test json-21.25 { + SELECT count(*) FROM json_tree(NULL); +} 0 +do_execsql_test json-21.26 { + WITH c(x) AS (VALUES(1),(2.0),(NULL),('three')) + SELECT json_group_array(x) FROM c; +} {[1,2.0,null,"three"]} +do_execsql_test json-21.27 { + WITH c(x,y) AS (VALUES('a',1),('b',2.0),('c',NULL),(NULL,'three'),('e','four')) + SELECT json_group_object(x,y) FROM c; +} {{{"a":1,"b":2.0,"c":null,:"three","e":"four"}}} + + + + + finish_test diff --git a/test/json102.test b/test/json102.test index f551c4b823..bfd5e7ed07 100644 --- a/test/json102.test +++ b/test/json102.test @@ -301,18 +301,27 @@ for {set i 0} {$i<100} {incr i} { # allowing them. The following tests verify that the problem is now # fixed. # -do_execsql_test json102-1401 { SELECT json_valid('{"x":01}') } 0 -do_execsql_test json102-1402 { SELECT json_valid('{"x":-01}') } 0 -do_execsql_test json102-1403 { SELECT json_valid('{"x":0}') } 1 -do_execsql_test json102-1404 { SELECT json_valid('{"x":-0}') } 1 -do_execsql_test json102-1405 { SELECT json_valid('{"x":0.1}') } 1 -do_execsql_test json102-1406 { SELECT json_valid('{"x":-0.1}') } 1 -do_execsql_test json102-1407 { SELECT json_valid('{"x":0.0000}') } 1 -do_execsql_test json102-1408 { SELECT json_valid('{"x":-0.0000}') } 1 -do_execsql_test json102-1409 { SELECT json_valid('{"x":01.5}') } 0 -do_execsql_test json102-1410 { SELECT json_valid('{"x":-01.5}') } 0 -do_execsql_test json102-1411 { SELECT json_valid('{"x":00}') } 0 -do_execsql_test json102-1412 { SELECT json_valid('{"x":-00}') } 0 +foreach {id j x0 x5} { + 1401 {'{"x":01}'} 0 0 + 1402 {'{"x":-01}'} 0 0 + 1403 {'{"x":0}'} 1 1 + 1404 {'{"x":-0}'} 1 1 + 1405 {'{"x":0.1}'} 1 1 + 1406 {'{"x":-0.1}'} 1 1 + 1407 {'{"x":0.0000}'} 1 1 + 1408 {'{"x":-0.0000}'} 1 1 + 1409 {'{"x":01.5}'} 0 0 + 1410 {'{"x":-01.5}'} 0 0 + 1411 {'{"x":00}'} 0 0 + 1412 {'{"x":-00}'} 0 0 + 1413 {'{"x":+0}'} 0 1 + 1414 {'{"x":+5}'} 0 1 + 1415 {'{"x":+5.5}'} 0 1 +} { + do_execsql_test json102-$id " + SELECT json_valid($j), NOT json_error_position($j); + " [list $x0 $x5] +} #------------------------------------------------------------------------ # 2017-04-10 ticket 6c9b5514077fed34551f98e64c09a10dc2fc8e16 diff --git a/test/json104.test b/test/json104.test index 56dd2738c3..c3c43d1e98 100644 --- a/test/json104.test +++ b/test/json104.test @@ -30,6 +30,48 @@ do_execsql_test json104-100 { } }'); } {{{"a":"z","c":{"d":"e"}}}} +do_execsql_test json104-101 { + SELECT json_patch('{ + "a": "b", + "c": { + "d": "e", + "f": "g" + } + }','{ + a:"z", + c: { + f: null + } + }'); +} {{{"a":"z","c":{"d":"e"}}}} +do_execsql_test json104-102 { + SELECT json_patch('{ + a: "b", + c: { + d: "e", + f: "g" + } + }','{ + "a":"z", + "c": { + "f": null + } + }'); +} {{{"a":"z","c":{"d":"e"}}}} +do_execsql_test json104-103 { + SELECT json_patch('{ + a: "b", + c: { + d: "e", + f: "g" + } + }','{ + a:"z", + c: { + f: null + } + }'); +} {{{"a":"z","c":{"d":"e"}}}} # This is the example from pages 4 and 5 of RFC-7396 diff --git a/test/json501.test b/test/json501.test new file mode 100644 index 0000000000..a37326973a --- /dev/null +++ b/test/json501.test @@ -0,0 +1,304 @@ +# 2023-04-27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests for the JSON5 enhancements to the +# JSON SQL functions extension to the SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json501 + +# From https://spec.json5.org/#introduction +# +#----------------------------------------------------------------------------- +# Summary of Features +# +# The following ECMAScript 5.1 features, which are not supported in JSON, have +# been extended to JSON5. +# +# Objects +# +# 1) Object keys may be an ECMAScript 5.1 IdentifierName. +# 2) Objects may have a single trailing comma. +# +# Arrays +# +# 3) Arrays may have a single trailing comma. +# +# Strings +# +# 4) Strings may be single quoted. +# 5) Strings may span multiple lines by escaping new line characters. +# 6) Strings may include character escapes. +# +# Numbers +# +# 7) Numbers may be hexadecimal. +# 8) Numbers may have a leading or trailing decimal point. +# 9) Numbers may be IEEE 754 positive infinity, negative infinity, and NaN. +# 10) Numbers may begin with an explicit plus sign. +# +# Comments +# +# 11) Single and multi-line comments are allowed. +# +# White Space +# +# 12) Additional white space characters are allowed. +#----------------------------------------------------------------------------- +# +# Test number in this file are of the form X.Y where X is one of the item +# numbers in the feature list above and Y is the test sequence number. +# + +############################################################################### +# 1) Object keys may be an ECMAScript 5.1 IdentifierName. +do_execsql_test 1.1 { + WITH c(x) AS (VALUES('{a:5,b:6}')) + SELECT x->>'a', json(x), json_valid(x), NOT json_error_position(x) FROM c; +} {5 {{"a":5,"b":6}} 0 1} +do_execsql_test 1.2 { + SELECT '[7,null,{a:5,b:6},[8,9]]'->>'$[2].b'; +} {6} +do_execsql_test 1.3 { + SELECT '{ $123 : 789 }'->>'$."$123"'; +} 789 +do_execsql_test 1.4 { + SELECT '{ _123$xyz : 789 }'->>'$."_123$xyz"'; +} 789 +do_execsql_test 1.5 { + SELECT '{ MNO_123$xyz : 789 }'->>'$."MNO_123$xyz"'; +} 789 + +do_execsql_test 1.6 { + SELECT json('{ MNO_123$xyz : 789 }'); +} [list {{"MNO_123$xyz":789}}] + +do_catchsql_test 1.10 { + SELECT json('{ MNO_123/xyz : 789 }'); +} {1 {malformed JSON}} + +do_execsql_test 1.11 { + SELECT '{ MNO_123æxyz : 789 }'->>'MNO_123æxyz'; +} {789} + +############################################################################### +# 2) Objects may have a single trailing comma. + +do_execsql_test 2.1 { + WITH c(x) AS (VALUES('{"a":5, "b":6, }')) + SELECT x->>'b', json(x), json_valid(x), NOT json_error_position(x) FROM c; +} {6 {{"a":5,"b":6}} 0 1} +do_execsql_test 2.2 { + SELECT '{a:5, b:6 , }'->>'b'; +} 6 +do_catchsql_test 2.3 { + SELECT '{a:5, b:6 ,, }'->>'b'; +} {1 {malformed JSON}} +do_catchsql_test 2.4 { + SELECT '{a:5, b:6, ,}'->>'b'; +} {1 {malformed JSON}} + +############################################################################### +# 3) Arrays may have a single trailing comma. + +do_execsql_test 3.1 { + WITH c(x) AS (VALUES('[5, 6,]')) + SELECT x->>1, json(x), json_valid(x), NOT json_error_position(x) FROM c; +} {6 {[5,6]} 0 1} +do_execsql_test 3.2 { + SELECT '[5, 6 , ]'->>1; +} 6 +do_catchsql_test 3.3 { + SELECT '[5, 6,,]'->>1; +} {1 {malformed JSON}} +do_catchsql_test 3.4 { + SELECT '[5, 6 , , ]'->>1; +} {1 {malformed JSON}} + +############################################################################### +# 4) Strings may be single quoted. + +do_execsql_test 4.1 { + WITH c(x) AS (VALUES('{"a": ''abcd''}')) + SELECT x->>'a', json(x), json_valid(x), NOT json_error_position(x) FROM c; +} {abcd {{"a":"abcd"}} 0 1} +do_execsql_test 4.2 { + SELECT '{b: 123, ''a'': ''ab\''cd''}'->>'a'; +} {ab'cd} + +############################################################################### +# 5) Strings may span multiple lines by escaping new line characters. + +do_execsql_test 5.1 { + WITH c(x) AS (VALUES('{a: "abc'||char(0x5c,0x0a)||'xyz"}')) + SELECT x->>'a', json(x), json_valid(x), NOT json_error_position(x) FROM c; +} {abcxyz {{"a":"abcxyz"}} 0 1} +do_execsql_test 5.2 { + SELECT ('{a: "abc'||char(0x5c,0x0d)||'xyz"}')->>'a'; +} {abcxyz} +do_execsql_test 5.3 { + SELECT ('{a: "abc'||char(0x5c,0x0d,0x0a)||'xyz"}')->>'a'; +} {abcxyz} +do_execsql_test 5.4 { + SELECT ('{a: "abc'||char(0x5c,0x2028)||'xyz"}')->>'a'; +} {abcxyz} +do_execsql_test 5.5 { + SELECT ('{a: "abc'||char(0x5c,0x2029)||'xyz"}')->>'a'; +} {abcxyz} + + +############################################################################### +# 6) Strings may include character escapes. + +do_execsql_test 6.1 { + SELECT ('{a: "abc'||char(0x5c,0x27)||'xyz"}')->>'a'; +} {abc'xyz} +do_execsql_test 6.2 { + SELECT ('{a: "abc'||char(0x5c,0x22)||'xyz"}')->>'a'; +} {abc"xyz} +do_execsql_test 6.3 { + SELECT ('{a: "abc'||char(0x5c,0x5c)||'xyz"}')->>'a'; +} {{abc\xyz}} +do_execsql_test 6.4 { + SELECT hex(('{a: "abc\bxyz"}')->>'a'); +} {6162630878797A} +do_execsql_test 6.5 { + SELECT hex(('{a: "abc\f\n\r\t\vxyz"}')->>'a'); +} {6162630C0A0D090B78797A} +do_execsql_test 6.6 { + SELECT hex(('{a: "abc\0xyz"}')->>'a'); +} {6162630078797A} +do_execsql_test 6.7 { + SELECT '{a: "abc\x35\x4f\x6Exyz"}'->>'a'; +} {abc5Onxyz} +do_execsql_test 6.8 { + SELECT '{a: "\x6a\x6A\x6b\x6B\x6c\x6C\x6d\x6D\x6e\x6E\x6f\x6F"}'->>'a'; +} {jjkkllmmnnoo} + +############################################################################### +# 7) Numbers may be hexadecimal. + +do_execsql_test 7.1 { + SELECT '{a: 0x0}'->>'a'; +} 0 +do_execsql_test 7.2 { + SELECT '{a: -0x0}'->>'a'; +} 0 +do_execsql_test 7.3 { + SELECT '{a: +0x0}'->>'a'; +} 0 +do_execsql_test 7.4 { + SELECT '{a: 0xabcdef}'->>'a'; +} 11259375 +do_execsql_test 7.5 { + SELECT '{a: -0xaBcDeF}'->>'a'; +} -11259375 +do_execsql_test 7.6 { + SELECT '{a: +0xABCDEF}'->>'a'; +} 11259375 + +############################################################################### +# 8) Numbers may have a leading or trailing decimal point. + +do_execsql_test 8.1 { + WITH c(x) AS (VALUES('{x: 4.}')) SELECT x->>'x', json(x) FROM c; +} {4.0 {{"x":4.0}}} +do_execsql_test 8.2 { + WITH c(x) AS (VALUES('{x: +4.}')) SELECT x->>'x', json(x) FROM c; +} {4.0 {{"x":4.0}}} +do_execsql_test 8.3 { + WITH c(x) AS (VALUES('{x: -4.}')) SELECT x->>'x', json(x) FROM c; +} {-4.0 {{"x":-4.0}}} +do_execsql_test 8.3 { + WITH c(x) AS (VALUES('{x: .5}')) SELECT x->>'x', json(x) FROM c; +} {0.5 {{"x":0.5}}} +do_execsql_test 8.4 { + WITH c(x) AS (VALUES('{x: -.5}')) SELECT x->>'x', json(x) FROM c; +} {-0.5 {{"x":-0.5}}} +do_execsql_test 8.5 { + WITH c(x) AS (VALUES('{x: +.5}')) SELECT x->>'x', json(x) FROM c; +} {0.5 {{"x":0.5}}} +do_execsql_test 8.6 { + WITH c(x) AS (VALUES('{x: 4.e0}')) SELECT x->>'x', json(x) FROM c; +} {4.0 {{"x":4.0e0}}} +do_execsql_test 8.7 { + WITH c(x) AS (VALUES('{x: +4.e1}')) SELECT x->>'x', json(x) FROM c; +} {40.0 {{"x":4.0e1}}} +do_execsql_test 8.8 { + WITH c(x) AS (VALUES('{x: -4.e2}')) SELECT x->>'x', json(x) FROM c; +} {-400.0 {{"x":-4.0e2}}} +do_execsql_test 8.9 { + WITH c(x) AS (VALUES('{x: .5e3}')) SELECT x->>'x', json(x) FROM c; +} {500.0 {{"x":0.5e3}}} +do_execsql_test 8.10 { + WITH c(x) AS (VALUES('{x: -.5e-1}')) SELECT x->>'x', json(x) FROM c; +} {-0.05 {{"x":-0.5e-1}}} +do_execsql_test 8.11 { + WITH c(x) AS (VALUES('{x: +.5e-2}')) SELECT x->>'x', json(x) FROM c; +} {0.005 {{"x":0.5e-2}}} + + +############################################################################### +# 9) Numbers may be IEEE 754 positive infinity, negative infinity, and NaN. + +do_execsql_test 9.1 { + WITH c(x) AS (VALUES('{x: +Infinity}')) SELECT x->>'x', json(x) FROM c; +} {Inf {{"x":9.0e999}}} +do_execsql_test 9.2 { + WITH c(x) AS (VALUES('{x: -Infinity}')) SELECT x->>'x', json(x) FROM c; +} {-Inf {{"x":-9.0e999}}} +do_execsql_test 9.3 { + WITH c(x) AS (VALUES('{x: Infinity}')) SELECT x->>'x', json(x) FROM c; +} {Inf {{"x":9.0e999}}} +do_execsql_test 9.4 { + WITH c(x) AS (VALUES('{x: NaN}')) SELECT x->>'x', json(x) FROM c; +} {{} {{"x":null}}} + +############################################################################### +# 10) Numbers may begin with an explicit plus sign. + +do_execsql_test 10.1 { + SELECT '{a: +123}'->'a'; +} 123 + +############################################################################### +# 11) Single and multi-line comments are allowed. + +do_execsql_test 11.1 { + SELECT ' /* abc */ { /*def*/ aaa /* xyz */ : // to the end of line + 123 /* xyz */ , /* 123 */ }'->>'aaa'; +} 123 + +############################################################################### +# 12) Additional white space characters are allowed. + +do_execsql_test 12.1 { + SELECT (char(0x09,0x0a,0x0b,0x0c,0x0d,0x20,0xa0,0x2028,0x2029) + || '{a: "xyz"}')->>'a'; +} xyz +do_execsql_test 12.2 { + SELECT ('{a:' || char(0x09,0x0a,0x0b,0x0c,0x0d,0x20,0xa0,0x2028,0x2029) + || '"xyz"}')->>'a'; +} xyz +do_execsql_test 12.3 { + SELECT (char(0x1680,0x2000,0x2001,0x2002,0x2003,0x2004,0x2005, + 0x2006,0x2007,0x2008,0x2009,0x200a,0x3000,0xfeff) + || '{a: "xyz"}')->>'a'; +} xyz +do_execsql_test 12.4 { + SELECT ('{a: ' ||char(0x1680,0x2000,0x2001,0x2002,0x2003,0x2004,0x2005, + 0x2006,0x2007,0x2008,0x2009,0x200a,0x3000,0xfeff) + || ' "xyz"}')->>'a'; +} xyz + + +finish_test diff --git a/test/json502.test b/test/json502.test new file mode 100644 index 0000000000..b5e570320d --- /dev/null +++ b/test/json502.test @@ -0,0 +1,25 @@ +# 2023-04-28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests for the JSON5 enhancements to the +# JSON SQL functions extension to the SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json502 + +do_execsql_test 1.1 { + CREATE TABLE t1(x JSON); + INSERT INTO t1(x) VALUES('{a:{b:{c:"hello",},},}'); + SELECT fullkey FROM t1, json_tree(x); +} {{$} {$.a} {$.a.b} {$.a.b.c}} + +finish_test diff --git a/test/misc1.test b/test/misc1.test index 44914156cb..83acc752af 100644 --- a/test/misc1.test +++ b/test/misc1.test @@ -592,6 +592,9 @@ do_test misc1-18.1 { set n [sqlite3_sleep 100] expr {$n>=100} } {1} +do_test misc1-18.2 { + sqlite3_sleep -100 +} {0} # 2014-01-10: In a CREATE TABLE AS, if one or more of the column names # are an empty string, that is still OK. diff --git a/test/notnull2.test b/test/notnull2.test index 12fffe28ce..7f68086810 100644 --- a/test/notnull2.test +++ b/test/notnull2.test @@ -28,24 +28,6 @@ do_execsql_test 1.0 { INSERT INTO t2 SELECT * FROM t1; } -proc do_vmstep_test {tn sql nstep {res {}}} { - uplevel [list do_execsql_test $tn.0 $sql $res] - - set vmstep [db status vmstep] - if {[string range $nstep 0 0]=="+"} { - set body "if {$vmstep<$nstep} { - error \"got $vmstep, expected more than [string range $nstep 1 end]\" - }" - } else { - set body "if {$vmstep>$nstep} { - error \"got $vmstep, expected less than $nstep\" - }" - } - - # set name "$tn.vmstep=$vmstep,expect=$nstep" - set name "$tn.1" - uplevel [list do_test $name $body {}] -} do_vmstep_test 1.1.1 { SELECT * FROM t1 LEFT JOIN t2 WHERE a=c AND d IS NULL; diff --git a/test/pragma.test b/test/pragma.test index 9ab332c3f1..5b45a74400 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -433,9 +433,9 @@ ifcapable attach { PRAGMA integrity_check } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} do_execsql_test pragma-3.9b { PRAGMA t2.integrity_check=t2; } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} @@ -447,79 +447,79 @@ Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2 PRAGMA integrity_check=1 } } {{*** in database t2 *** -Page 4 is never used}} +Page 4: never used}} do_test pragma-3.11 { execsql { PRAGMA integrity_check=5 } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2}} do_test pragma-3.12 { execsql { PRAGMA integrity_check=4 } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2}} do_test pragma-3.13 { execsql { PRAGMA integrity_check=3 } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used}} +Page 4: never used +Page 5: never used +Page 6: never used}} do_test pragma-3.14 { execsql { PRAGMA integrity_check(2) } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used}} +Page 4: never used +Page 5: never used}} do_test pragma-3.15 { execsql { ATTACH 'testerr.db' AS t3; PRAGMA integrity_check } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} do_test pragma-3.16 { execsql { PRAGMA integrity_check(10) } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2}} do_test pragma-3.17 { execsql { PRAGMA integrity_check=8 } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** -Page 4 is never used -Page 5 is never used}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** +Page 4: never used +Page 5: never used}} do_test pragma-3.18 { execsql { PRAGMA integrity_check=4 } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2}} } do_test pragma-3.19 { catch {db close} @@ -2060,4 +2060,17 @@ ifcapable !has_codec { } {0 {{database disk image is malformed}}} } database_never_corrupt + +# 2023-03-27. Register allocation issue in integrity_check discovered +# by new assert() statements added in [6f8b97f31a4c8552]. +# dbsqlfuzz dc9ab26037cf5ef797d28cd1ae0855ade584216d +# tag-20230327-1 +# +reset_db +do_execsql_test 25.0 { + CREATE TABLE t1(a INT, b AS (a*2) NOT NULL); + CREATE TEMP TABLE t2(a PRIMARY KEY, b, c UNIQUE) WITHOUT ROWID; + CREATE UNIQUE INDEX t2x ON t2(c,b); + PRAGMA integrity_check; +} ok finish_test diff --git a/test/printf.test b/test/printf.test index 445470fc07..e4beb12dd0 100644 --- a/test/printf.test +++ b/test/printf.test @@ -3786,4 +3786,42 @@ do_execsql_test printf-16.1 { SELECT printf('%.*g',2147483647,0.01); } {0.01} +# 2023-02-23 https://sqlite.org/forum/forumpost/d1387c3979c7f557 +# Loss of precision when doing floating-point to decimal +# conversions on values that have no factional part. +# +do_execsql_test printf-17.1 { + SELECT format('%!.20g', 13.0); +} 13.0 +do_execsql_test printf-17.2 { + SELECT format('%.3e', 199990000.0); +} 2.000e+08 +do_execsql_test printf-17.3 { + SELECT format('%.3f', 199990000.0); +} 199990000.000 +do_execsql_test printf-17.4 { + SELECT format('%.3g', 199990000.0); +} 2e+08 +do_execsql_test printf-17.5 { + SELECT format('%.4e', 199990000.0); +} 1.9999e+08 +do_execsql_test printf-17.6 { + SELECT format('%.4f', 199990000.0); +} 199990000.0000 +do_execsql_test printf-17.7 { + SELECT format('%.4g', 199990000.0); +} 2e+08 +do_execsql_test printf-17.8 { + SELECT format('%.5e', 199990000.0); +} 1.99990e+08 +do_execsql_test printf-17.9 { + SELECT format('%.5f', 199990000.0); +} 199990000.00000 +do_execsql_test printf-17.10 { + SELECT format('%.5g', 199990000.0); +} 1.9999e+08 +do_execsql_test printf-17.11 { + SELECT format('%.30f',1.0000000000000000076e-50); +} 0.000000000000000000000000000000 + finish_test diff --git a/test/pushdown.test b/test/pushdown.test index ca816fb213..b9ee48c4ac 100644 --- a/test/pushdown.test +++ b/test/pushdown.test @@ -122,5 +122,66 @@ do_execsql_test 3.4 { } { 2 0 0 } - + +# 2023-02-22 https://sqlite.org/forum/forumpost/bcc4375032 +# Performance regression caused by check-in [1ad41840c5e0fa70] from 2022-11-25. +# That check-in added a new restriction on push-down. The new restriction is +# no longer necessary after check-in [27655c9353620aa5] from 2022-12-14. +# +do_execsql_test 3.5 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INT, b INT, c TEXT, PRIMARY KEY(a,b)) WITHOUT ROWID; + INSERT INTO t1(a,b,c) VALUES + (1,100,'abc'), + (2,200,'def'), + (3,300,'abc'); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(a INT, b INT, c TEXT, PRIMARY KEY(a,b)) WITHOUT ROWID; + INSERT INTO t2(a,b,c) VALUES + (1,110,'efg'), + (2,200,'hij'), + (3,330,'klm'); + CREATE VIEW v3 AS + SELECT a, b, c FROM t1 + UNION ALL + SELECT a, b, 'xyz' FROM t2; + SELECT * FROM v3 WHERE a=2 AND b=200; +} {2 200 def 2 200 xyz} +do_eqp_test 3.6 { + SELECT * FROM v3 WHERE a=2 AND b=200; +} { + QUERY PLAN + |--CO-ROUTINE v3 + | `--COMPOUND QUERY + | |--LEFT-MOST SUBQUERY + | | `--SEARCH t1 USING PRIMARY KEY (a=? AND b=?) + | `--UNION ALL + | `--SEARCH t2 USING PRIMARY KEY (a=? AND b=?) + `--SCAN v3 +} +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# We want both arms of the compound subquery to use the +# primary key. + +# The following is a test of the count-of-view optimization. This does +# not have anything to do with push-down. It is here because this is a +# convenient place to put the test. +# +do_execsql_test 3.7 { + SELECT count(*) FROM v3; +} 6 +do_eqp_test 3.8 { + SELECT count(*) FROM v3; +} { + QUERY PLAN + |--SCAN CONSTANT ROW + |--SCALAR SUBQUERY xxxxxx + | `--SCAN t1 + `--SCALAR SUBQUERY xxxxxx + `--SCAN t2 +} +# ^^^^^^^^^^^^^^^^^^^^ +# The query should be converted into: +# SELECT (SELECT count(*) FROM t1)+(SELECT count(*) FROM t2) + finish_test diff --git a/test/returning1.test b/test/returning1.test index 326af800e1..6c098dc256 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -212,13 +212,13 @@ do_execsql_test 10.2 { END; } -do_catchsql_test 10.3 { +do_catchsql_test 10.3a { INSERT INTO t1(a, b) VALUES(1234, 5678) RETURNING rowid; -} {1 {no such column: rowid}} +} {1 {no such column: new.rowid}} -do_catchsql_test 10.3 { +do_catchsql_test 10.3b { UPDATE t1 SET a='z' WHERE b='y' RETURNING rowid; -} {1 {no such column: rowid}} +} {1 {no such column: new.rowid}} do_execsql_test 10.4 { SELECT * FROM log; @@ -408,4 +408,35 @@ do_execsql_test 17.0 { UPDATE bug SET x=NULL WHERE id = 20 RETURNING quote(x), x IS NULL; } {NULL 1} +# 2023-03-08 https://sqlite.org/forum/forumpost/f5a2b1db87 +# NULL pointer dereference following an error. +# +do_execsql_test 18.0 { + CREATE TABLE v0(c1 INT); + CREATE VIEW view_2(c1) AS SELECT CASE WHEN c1 COLLATE TRUE THEN TRUE ELSE TRUE END FROM v0; + CREATE TRIGGER x1 INSTEAD OF INSERT ON view_2 BEGIN SELECT true; END; +} +do_catchsql_test 18.1 { + INSERT INTO view_2 DEFAULT VALUES RETURNING *; +} {1 {no such collation sequence: TRUE}} + +# 2023-03-16 +# https://sqlite.org/forum/forumpost/c99d6e0329 +# ticket d15b3a4ea901ef0d +# ticket 89d259d45b855a0d +# +# A RETURNING clause on an IF NOT EXISTS trigger does not generate +# an error if the trigger already exists. +# +do_execsql_test 19.0 { + DROP TABLE IF EXISTS t1;CREATE TABLE t1(a); + CREATE TRIGGER r1 AFTER UPDATE ON t1 BEGIN VALUES(0); END; +} {} +do_catchsql_test 19.1 { + CREATE TRIGGER IF NOT EXISTS r1 AFTER DELETE ON t1 BEGIN + INSERT INTO t1(a) VALUES (1) RETURNING FALSE; + INSERT INTO t1(a) VALUES (2) RETURNING TRUE; + END; +} {0 {}} + finish_test diff --git a/test/scanstatus.test b/test/scanstatus.test index fa00a356bc..549e7fd3c8 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -45,12 +45,20 @@ proc do_scanstatus_test {tn res} { uplevel [list do_test $tn [list set {} $ret] [list {*}$res]] } -do_execsql_test 1.1 { SELECT count(*) FROM t1, t2; } 6 -do_scanstatus_test 1.2 { +do_execsql_test 1.1a { SELECT count(*) FROM t1, t2; } 6 +do_scanstatus_test 1.1b { nLoop 1 nVisit 2 nEst 1048576.0 zName t1 zExplain {SCAN t1} nLoop 2 nVisit 6 nEst 1048576.0 zName t2 zExplain {SCAN t2} } +sqlite3_db_config db STMT_SCANSTATUS 0 + +do_execsql_test 1.2a { SELECT count(*) FROM t1, t2; } 6 +do_scanstatus_test 1.2b { +} + +sqlite3_db_config db STMT_SCANSTATUS 1 + do_execsql_test 1.3 { ANALYZE; SELECT count(*) FROM t1, t2; @@ -94,6 +102,7 @@ do_scanstatus_test 1.10 { # Try a few different types of scans. # reset_db +sqlite3_db_config db STMT_SCANSTATUS 1 do_execsql_test 2.1 { CREATE TABLE x1(i INTEGER PRIMARY KEY, j); INSERT INTO x1 VALUES(1, 'one'); @@ -277,6 +286,7 @@ do_scanstatus_test 4.2.2 { # Further tests of different scan types. # reset_db +sqlite3_db_config db STMT_SCANSTATUS 1 proc tochar {i} { set alphabet {a b c d e f g h i j k l m n o p q r s t u v w x y z} return [lindex $alphabet [expr $i % [llength $alphabet]]] @@ -365,6 +375,7 @@ do_eqp_test 5.5.1 { } { QUERY PLAN |--SCAN t3 + |--BLOOM FILTER ON t1 (c=?) `--SEARCH t1 USING AUTOMATIC COVERING INDEX (c=?) } do_execsql_test 5.5.2 { diff --git a/test/scanstatus2.test b/test/scanstatus2.test index 497cbe67d8..fb62af2103 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -19,6 +19,8 @@ ifcapable !scanstatus { return } +sqlite3_db_config db STMT_SCANSTATUS 1 + do_execsql_test 1.0 { CREATE TABLE t1(a, b); CREATE TABLE t2(x, y); @@ -141,6 +143,7 @@ QUERY (nCycle=nnn) #------------------------------------------------------------------------- ifcapable fts5 { reset_db + sqlite3_db_config db STMT_SCANSTATUS 1 do_execsql_test 2.0 { CREATE VIRTUAL TABLE ft USING fts5(a); INSERT INTO ft VALUES('abc'); @@ -158,6 +161,7 @@ QUERY (nCycle=nnn) #------------------------------------------------------------------------- reset_db +sqlite3_db_config db STMT_SCANSTATUS 1 do_execsql_test 3.0 { CREATE TABLE x1(a, b); CREATE TABLE x2(c, d); @@ -173,11 +177,13 @@ do_graph_test 2.1 { QUERY (nCycle=nnn) --SCAN x1 (nCycle=nnn) --CREATE AUTOMATIC INDEX ON x2(c, d) (nCycle=nnn) +--BLOOM FILTER ON x2 (c=?) --SEARCH x2 USING AUTOMATIC COVERING INDEX (c=?) (nCycle=nnn) } #------------------------------------------------------------------------- reset_db +sqlite3_db_config db STMT_SCANSTATUS 1 do_execsql_test 4.0 { CREATE TABLE rt1 (id INTEGER PRIMARY KEY, x1, x2); CREATE TABLE rt2 (id, x1, x2); @@ -189,6 +195,7 @@ do_graph_test 4.1 { QUERY (nCycle=nnn) --SCAN rt1 (nCycle=nnn) --CREATE AUTOMATIC INDEX ON rt2(x1, id, x2) (nCycle=nnn) +--BLOOM FILTER ON rt2 (x1=?) --SEARCH rt2 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) } @@ -198,6 +205,7 @@ do_graph_test 4.2 { QUERY (nCycle=nnn) --SCAN rt1 (nCycle=nnn) --CREATE AUTOMATIC INDEX ON rt2(x1, id) (nCycle=nnn) +--BLOOM FILTER ON rt2 (x1=?) --SEARCH rt2 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) } @@ -215,6 +223,7 @@ do_graph_test 4.4 { QUERY (nCycle=nnn) --SCAN rt1 (nCycle=nnn) --CREATE AUTOMATIC INDEX ON rt2(x1, id) WHERE (nCycle=nnn) +--BLOOM FILTER ON rt2 (x1=?) --SEARCH rt2 USING AUTOMATIC PARTIAL COVERING INDEX (x1=?) (nCycle=nnn) } @@ -229,9 +238,38 @@ QUERY (nCycle=nnn) ----USE TEMP B-TREE FOR GROUP BY --SCAN rt1 (nCycle=nnn) --CREATE AUTOMATIC INDEX ON v1(x1, cnt) (nCycle=nnn) +--BLOOM FILTER ON v1 (x1=?) --SEARCH v1 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) } +#------------------------------------------------------------------------- +reset_db + +ifcapable trace { + do_execsql_test 5.0 { + CREATE TABLE t1(x, y); + CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN + SELECT 1; + END; + INSERT INTO t1 VALUES(1, 2); + } + + proc trace {stmt sql} { + array set A [sqlite3_stmt_scanstatus -flags complex [format %x $stmt] 0] + lappend ::trace_explain $A(zExplain) + } + db trace_v2 trace + + set ::trace_explain [list] + do_execsql_test 5.1 { + DELETE FROM t1 WHERE x=1; + } + + do_test 5.2 { + set ::trace_explain + } {{SCAN t1} {SCAN t1} {SCAN t1}} +} + finish_test diff --git a/test/seekscan1.test b/test/seekscan1.test index e68d71449d..0e53c19166 100644 --- a/test/seekscan1.test +++ b/test/seekscan1.test @@ -24,6 +24,7 @@ do_execsql_test 1.0 { ANALYZE; } + do_execsql_test 1.1 { SELECT a,b,c FROM t1 WHERE b IN (234, 345) AND c BETWEEN 6 AND 6.5 AND a='abc' @@ -59,5 +60,9 @@ do_execsql_test 1.4 { abc 345 7 } +do_execsql_test 1.5 { + SELECT a,b,c FROM t1 WHERE b IN (235, 345) AND c<=3 AND a='abc' ORDER BY a, b; +} + finish_test diff --git a/test/selectC.test b/test/selectC.test index 63851ca5d7..42fa1d11b8 100644 --- a/test/selectC.test +++ b/test/selectC.test @@ -230,6 +230,11 @@ do_execsql_test selectC-4.2 { select a from (select distinct a, b from t_distinct_bug) } {1 1 1} +do_execsql_test selectC-4.2b { + CREATE VIEW v42b AS SELECT DISTINCT a, b FROM t_distinct_bug; + SELECT a FROM v42b; +} {1 1 1} + do_execsql_test selectC-4.3 { select a, udf() from (select distinct a, b from t_distinct_bug) } {1 1 1 2 1 3} diff --git a/test/selectH.test b/test/selectH.test new file mode 100644 index 0000000000..a97679bda2 --- /dev/null +++ b/test/selectH.test @@ -0,0 +1,118 @@ +# 2023-02-16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the omit-unused-subquery-column optimization. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix selectH + +do_execsql_test 1.1 { + CREATE TABLE t1( + c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, + c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, + c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, + c30, c31, c32, c33, c34, c35, c36, c37, c38, c39, + c40, c41, c42, c43, c44, c45, c46, c47, c48, c49, + c50, c51, c52, c53, c54, c55, c56, c57, c58, c59, + c60, c61, c62, c63, c64, c65 + ); + INSERT INTO t1 VALUES( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65 + ); + CREATE INDEX t1c60 ON t1(c60); +} + +# The SQL counter(N) function adjusts the value of the global +# TCL variable ::selectH_cnt by the value N and returns the new +# value. By putting calls to counter(N) as unused columns in a +# view or subquery, we can check to see if the counter gets incremented, +# and if not that means that the unused column was omitted. +# +unset -nocomplain selectH_cnt +set selectH_cnt 0 +proc selectH_counter {amt} { + global selectH_cnt + incr selectH_cnt $amt + return $selectH_cnt +} +db func counter selectH_counter + +do_execsql_test 1.2 { + SELECT DISTINCT c44 FROM ( + SELECT c0 AS a, *, counter(1) FROM t1 + UNION ALL + SELECT c1 AS a, *, counter(1) FROM t1 + ) WHERE c60=60; +} {44} +do_test 1.3 { + set ::selectH_cnt +} {0} + +do_execsql_test 2.1 { + SELECT a FROM ( + SELECT counter(1) AS cnt, c15 AS a, *, c62 AS b FROM t1 + UNION ALL + SELECT counter(1) AS cnt, c16 AS a, *, c61 AS b FROM t1 + ORDER BY b + ); +} {16 15} +do_test 2.2 { + set ::selectH_cnt +} {0} + +do_execsql_test 3.1 { + CREATE VIEW v1 AS + SELECT c16 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c17 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c18 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c19 AS a, *, counter(1) AS x FROM t1; + SELECT count(*) FROM v1 WHERE c60=60; +} {4} +do_test 3.2 { + set ::selectH_cnt +} {0} +do_execsql_test 3.3 { + SELECT count(a) FROM v1 WHERE c60=60; +} {4} +do_execsql_test 3.4 { + SELECT a FROM v1 WHERE c60=60; +} {16 17 18 19} +do_test 3.5 { + set ::selectH_cnt +} {0} +do_execsql_test 3.6 { + SELECT x FROM v1 WHERE c60=60; +} {1 2 3 4} +do_test 3.7 { + set ::selectH_cnt +} {4} + +# 2023-02-25 dbsqlfuzz bf1d3ed6e0e0dd8766027797d43db40c776d2b15 +# +do_execsql_test 4.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + SELECT 1 FROM (SELECT DISTINCT name COLLATE rtrim FROM sqlite_schema + UNION ALL SELECT a FROM t1); +} 1 + +finish_test diff --git a/test/shell1.test b/test/shell1.test index fbbda0348d..7a32c7b1bb 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -175,6 +175,16 @@ do_test shell1-1.16.1 { set x [catchcmd "-version test.db" ""] } {/3.[0-9.]+ 20\d\d-[01]\d-\d\d \d\d:\d\d:\d\d [0-9a-f]+/} +# Handle no-more-options option +forcedelete ./--db +do_test shell1-1.17.1 { + catchcmd {-- --db "CREATE TABLE T(c1);"} +} {0 {}} +do_test shell1-1.17.2 { + catchcmd {-- --db "SELECT name from sqlite_schema;"} +} {0 T} +forcedelete ./--db + #---------------------------------------------------------------------------- # Test cases shell1-2.*: Basic "dot" command token parsing. # diff --git a/test/shell2.test b/test/shell2.test index aba04a1490..3fad4bd665 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -216,5 +216,51 @@ do_test shell2-1.4.9 { done 2}} +# Verify that generate_series stays sane near 64-bit range boundaries. +# See overflow report at https://sqlite.org/forum/forumpost/5d34ce5280 +do_test shell2-1.4.10 { + set res [catchcmd :memory: [string trim { + SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); + SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); + SELECT avg(rowid),min(value),max(value) FROM generate_series( + -9223372036854775808,9223372036854775807,1085102592571150095); + SELECT * FROM generate_series(-9223372036854775808,9223372036854775807, + 9223372036854775807); + SELECT value,rowid FROM generate_series(-4611686018427387904, + 4611686018427387904, 4611686018427387904) ORDER BY value DESC; + SELECT * FROM generate_series(0,-2,-1); + SELECT * FROM generate_series(0,-2); + SELECT * FROM generate_series(0,2) LIMIT 3;}]] +} {0 {9223372036854775807 +9223372036854775807 +9.5|-9223372036854775808|9223372036854775807 +-9223372036854775808 +-1 +9223372036854775806 +4611686018427387904|3 +0|2 +-4611686018427387904|1 +0 +-1 +-2 +0 +1 +2}} + +# Bug discovered while messing around, .import hangs with +# bit 7 set in column separator. +do_test shell2-1.4.11 { + forcedelete dummy.csv + set df [open dummy.csv w] + puts $df dog,cat + close $df + set res [catchcmd :memory: [string trim { + CREATE TABLE t(line text); +.mode ascii +.separator "\377" "\n" +.import dummy.csv t + SELECT count(*) FROM t;}]] +} {0 1} + finish_test diff --git a/test/skipscan1.test b/test/skipscan1.test index 94062fb17e..bd5b83d34c 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -419,4 +419,14 @@ do_execsql_test skipscan1-4.10 { AND a <= 10; } {3} +# 2023-03-24 https://sqlite.org/forum/forumpost/8cc1dc0fe9 +# +reset_db +do_execsql_test skipscan1-5.0 { + CREATE TABLE t1(a TEXT, UNIQUE(a,a,a)); + INSERT INTO t1 VALUES (hex(zeroblob(241))),(1),(2),(3); + ANALYZE; + SELECT max(a) FROM t1 WHERE a IN t1; +} {3} + finish_test diff --git a/test/speedtest1.c b/test/speedtest1.c index 12c8d69b5b..6aff89b376 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -12,6 +12,7 @@ static const char zHelp[] = " --checkpoint Run PRAGMA wal_checkpoint after each test case\n" " --exclusive Enable locking_mode=EXCLUSIVE\n" " --explain Like --sqlonly but with added EXPLAIN keywords\n" + " --fullfsync Enable fullfsync=TRUE\n" " --heap SZ MIN Memory allocator uses SZ bytes & min allocation MIN\n" " --incrvacuum Enable incremenatal vacuum mode\n" " --journal M Set the journal_mode to M\n" @@ -39,6 +40,7 @@ static const char zHelp[] = " --size N Relative test size. Default=100\n" " --strict Use STRICT table where appropriate\n" " --stats Show statistics at the end\n" + " --stmtscanstatus Activate SQLITE_DBCONFIG_STMT_SCANSTATUS\n" " --temp N N from 0 to 9. 0: no temp table. 9: all temp tables\n" " --testset T Run test-set T (main, cte, rtree, orm, fp, debug)\n" " --trace Turn on SQL tracing\n" @@ -100,6 +102,7 @@ static struct Global { int nRepeat; /* Repeat selects this many times */ int doCheckpoint; /* Run PRAGMA wal_checkpoint after each trans */ int nReserve; /* Reserve bytes */ + int stmtScanStatus; /* True to activate Stmt ScanStatus reporting */ int doBigTransactions; /* Enable transactions on tests 410 and 510 */ const char *zWR; /* Might be WITHOUT ROWID */ const char *zNN; /* Might be NOT NULL */ @@ -2201,6 +2204,7 @@ int main(int argc, char **argv){ int doAutovac = 0; /* True for --autovacuum */ int cacheSize = 0; /* Desired cache size. 0 means default */ int doExclusive = 0; /* True for --exclusive */ + int doFullFSync = 0; /* True for --fullfsync */ int nHeap = 0, mnHeap = 0; /* Heap size from --heap */ int doIncrvac = 0; /* True for --incrvacuum */ const char *zJMode = 0; /* Journal mode */ @@ -2265,6 +2269,8 @@ int main(int argc, char **argv){ cacheSize = integerValue(argv[++i]); }else if( strcmp(z,"exclusive")==0 ){ doExclusive = 1; + }else if( strcmp(z,"fullfsync")==0 ){ + doFullFSync = 1; }else if( strcmp(z,"checkpoint")==0 ){ g.doCheckpoint = 1; }else if( strcmp(z,"explain")==0 ){ @@ -2391,6 +2397,8 @@ int main(int argc, char **argv){ }else if( strcmp(z,"reserve")==0 ){ ARGC_VALUE_CHECK(1); g.nReserve = atoi(argv[++i]); + }else if( strcmp(z,"stmtscanstatus")==0 ){ + g.stmtScanStatus = 1; }else if( strcmp(z,"without-rowid")==0 ){ if( strstr(g.zWR,"WITHOUT")!=0 ){ /* no-op */ @@ -2474,6 +2482,9 @@ int main(int argc, char **argv){ if( g.nReserve>0 ){ sqlite3_file_control(g.db, 0, SQLITE_FCNTL_RESERVE_BYTES, &g.nReserve); } + if( g.stmtScanStatus ){ + sqlite3_db_config(g.db, SQLITE_DBCONFIG_STMT_SCANSTATUS, 1, 0); + } /* Set database connection options */ sqlite3_create_function(g.db, "random", 0, SQLITE_UTF8, 0, randomFunc, 0, 0); @@ -2504,7 +2515,11 @@ int main(int argc, char **argv){ if( cacheSize ){ speedtest1_exec("PRAGMA cache_size=%d", cacheSize); } - if( noSync ) speedtest1_exec("PRAGMA synchronous=OFF"); + if( noSync ){ + speedtest1_exec("PRAGMA synchronous=OFF"); + }else if( doFullFSync ){ + speedtest1_exec("PRAGMA fullfsync=ON"); + } if( doExclusive ){ speedtest1_exec("PRAGMA locking_mode=EXCLUSIVE"); } diff --git a/test/subtype1.test b/test/subtype1.test index d15f737bc2..4d8e68e6af 100644 --- a/test/subtype1.test +++ b/test/subtype1.test @@ -53,7 +53,25 @@ do_execsql_test subtype1-231 { WITH t4(a) AS NOT MATERIALIZED (SELECT json(1)) SELECT subtype(a) FROM t4; } {0} - +# 2023-03-01 +# https://sqlite.org/forum/forumpost/37dd14a538 +# +# Additional tests to show that subtypes do not traverse subquery boundaries. +# +do_execsql_test subtype1-300 { + CREATE TABLE t0(c0); + INSERT INTO t0 VALUES ('1'); + CREATE VIEW v0(c0) AS SELECT CASE WHEN 1 THEN json_patch('1', '1') END + FROM t0 GROUP BY t0.c0; + SELECT * FROM v0 WHERE json_quote(v0.c0) != '1'; +} {1} +do_execsql_test subtype1-310 { + SELECT *, json_quote(y) FROM (SELECT +json('1') AS y); +} {1 {"1"}} +do_execsql_test subtype1-320 { + SELECT *, json_quote(y) FROM (SELECT +json('1') AS y) + WHERE json_quote(y)='"1"'; +} {1 {"1"}} finish_test diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 0a7e3bc245..8e90c549fa 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -61,7 +61,7 @@ do_execsql_test tabfunc01-1.8 { } {30 25 20 15 10 5 0} do_execsql_test tabfunc01-1.9 { SELECT rowid, * FROM generate_series(0,32,5) ORDER BY value DESC; -} {1 30 2 25 3 20 4 15 5 10 6 5 7 0} +} {7 30 6 25 5 20 4 15 3 10 2 5 1 0} do_execsql_test tabfunc01-1.10 { SELECT rowid, * FROM generate_series(0,32,5) ORDER BY +value DESC; } {7 30 6 25 5 20 4 15 3 10 2 5 1 0} @@ -270,6 +270,7 @@ do_test tabfunc01-750 { } {5.0 x5 | 7.0 x7 | 13.0 x13 | 17.0 x17 | 23.0 x23 |} # ticket https://www.sqlite.org/src/info/2ae0c599b735d59e +# Verification of testtag-20230227a do_test tabfunc01-751 { db eval { SELECT aa.value, bb.value, '|' diff --git a/test/tester.tcl b/test/tester.tcl index 4658590cf4..021830aa95 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -949,6 +949,29 @@ proc normalize_list {L} { set L2 } +# Run SQL and verify that the number of "vmsteps" required is greater +# than or less than some constant. +# +proc do_vmstep_test {tn sql nstep {res {}}} { + uplevel [list do_execsql_test $tn.0 $sql $res] + + set vmstep [db status vmstep] + if {[string range $nstep 0 0]=="+"} { + set body "if {$vmstep<$nstep} { + error \"got $vmstep, expected more than [string range $nstep 1 end]\" + }" + } else { + set body "if {$vmstep>$nstep} { + error \"got $vmstep, expected less than $nstep\" + }" + } + + # set name "$tn.vmstep=$vmstep,expect=$nstep" + set name "$tn.1" + uplevel [list do_test $name $body {}] +} + + # Either: # # do_execsql_test TESTNAME SQL ?RES? diff --git a/test/testrunner.tcl b/test/testrunner.tcl index a71b055e4c..4e21ce08ab 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -71,26 +71,32 @@ of sub-processes the test script uses to run tests. # switch. # proc guess_number_of_cores {} { - set ret 4 + if {[catch {number_of_cores} ret]} { + set ret 4 - if {$::tcl_platform(os)=="Darwin"} { - set cmd "sysctl -n hw.logicalcpu" - } else { - set cmd "nproc" - } - catch { - set fd [open "|$cmd" r] - set ret [gets $fd] - close $fd - set ret [expr $ret] + if {$::tcl_platform(os)=="Darwin"} { + set cmd "sysctl -n hw.logicalcpu" + } else { + set cmd "nproc" + } + catch { + set fd [open "|$cmd" r] + set ret [gets $fd] + close $fd + set ret [expr $ret] + } } return $ret } proc default_njob {} { set nCore [guess_number_of_cores] - set nHelper [expr int($nCore*0.75)] - expr $nHelper>0 ? $nHelper : 1 + if {$nCore<=2} { + set nHelper 1 + } else { + set nHelper [expr int($nCore*0.5)] + } + return $nHelper } #------------------------------------------------------------------------- @@ -878,6 +884,9 @@ proc run_testset {} { sqlite3 trdb $TRG(dbname) trdb timeout $TRG(timeout) set tm [lindex [time { make_new_testset }] 0] +if {$TRG(nJob)>1} { + puts "splitting work across $TRG(nJob) cores" +} puts "built testset in [expr $tm/1000]ms.." run_testset trdb close diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test index 868f36c3a5..ba9fdc7027 100644 --- a/test/tkt-99378177930f87bd.test +++ b/test/tkt-99378177930f87bd.test @@ -176,4 +176,21 @@ do_execsql_test tkt-99378-310 { ); } {1 2} +# 2023-03-04 https://sqlite.org/forum/forumpost/a68313d054 +# +# See also indexexpr1-2200 added on 2023-03-18. +# +do_execsql_test tkt-99378-400 { + DROP TABLE t1; + CREATE TABLE t0(w); + INSERT INTO t0(w) VALUES(1); + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES(1); + CREATE INDEX t1x ON t1(x > 0); + CREATE VIEW t2(y) AS SELECT avg(w) FROM t0 GROUP BY w>1; + CREATE VIEW t3(z) AS SELECT count(*) FROM t2 WHERE y BETWEEN 0 and 0; + SELECT count(*) FROM t1 NOT INDEXED WHERE (SELECT z FROM t3); + SELECT count(*) FROM t1 INDEXED BY t1x WHERE (SELECT z FROM t3); +} {0 0} + finish_test diff --git a/test/update.test b/test/update.test index 66efd10ec8..47b6029bee 100644 --- a/test/update.test +++ b/test/update.test @@ -732,4 +732,37 @@ do_execsql_test update-20.30 { PRAGMA integrity_check; } {ok} +# 2023-03-16 https://sqlite.org/forum/forumpost/0007d1fdb1 +# A subquery in the WHERE clause of an UPDATE and behind a +# short-circuit evaluation caused problems because multi-row +# single-pass was selected. +# +# Similar problem for DELETE tested by delete-12.0. +# https://sqlite.org/src/info/73f0036f045bf371 +# +reset_db +do_execsql_test update-21.1 { + CREATE TABLE t1 (vkey INTEGER, c5 INTEGER); + INSERT INTO t1 VALUES(3,NULL),(6,-54); +} +db null NULL +do_execsql_test update-21.2 { + BEGIN; + UPDATE t1 SET vkey = 100 WHERE c5 is null; + SELECT * FROM t1 ORDER BY vkey, c5; + ROLLBACK; +} {6 -54 100 NULL} +do_execsql_test update-21.3 { + BEGIN; + UPDATE t1 SET vkey = 100 WHERE NOT (-10*(select min(vkey) from t1) >= c5); + SELECT * FROM t1 ORDER BY vkey, c5; + ROLLBACK; +} {3 NULL 6 -54} +do_execsql_test update-21.4 { + BEGIN; + UPDATE t1 SET vkey = 100 WHERE c5 is null OR NOT (-10*(select min(vkey) from t1) >= c5); + SELECT * FROM t1 ORDER BY vkey, c5; + ROLLBACK; +} {6 -54 100 NULL} + finish_test diff --git a/test/vt02.c b/test/vt02.c index f83fc9af95..ddad136fba 100644 --- a/test/vt02.c +++ b/test/vt02.c @@ -177,8 +177,6 @@ #include #endif -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Forward declarations */ typedef struct vt02_vtab vt02_vtab; typedef struct vt02_cur vt02_cur; @@ -381,7 +379,7 @@ static int vt02Filter( sqlite3_int64 v; pVal = 0; if( sqlite3_vtab_in_first(0, &pVal)!=SQLITE_MISUSE - || sqlite3_vtab_in_first(argv[iArg], &pVal)!=SQLITE_MISUSE + || sqlite3_vtab_in_first(argv[iArg], &pVal)!=SQLITE_ERROR ){ vt02ErrMsg(pCursor->pVtab, "unexpected success from sqlite3_vtab_in_first()"); @@ -420,7 +418,7 @@ static int vt02Filter( } if( bUseOffset ){ int nSkip = sqlite3_value_int(argv[iArg]); - while( nSkip-- > 0 ) vt02Next(pCursor); + while( nSkip-- > 0 && pCur->iiEof ) vt02Next(pCursor); } return SQLITE_OK; @@ -1001,10 +999,6 @@ static void vt02CoreInit(sqlite3 *db){ sqlite3_create_module(db, "vt02pkabcd", &vt02Module, (void*)zPkABCDSchema); } -#else -# define vt02CoreInit(db) -#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ - #ifdef TH3_VERSION static void vt02_init(th3state *p, int iDb, char *zArg){ vt02CoreInit(th3dbPointer(p, iDb)); diff --git a/test/window1.test b/test/window1.test index 0f22829847..783a739e3f 100644 --- a/test/window1.test +++ b/test/window1.test @@ -2210,4 +2210,157 @@ do_execsql_test 72.1 { SELECT * FROM v1 WHERE true; } {1 0} +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 72.0 { + CREATE TABLE t0(c0); + INSERT INTO t0(c0) VALUES (0); + CREATE VIEW v0(c0) AS SELECT TOTAL(0) OVER (PARTITION BY t0.c0) FROM t0; +} +do_execsql_test 72.1 { + SELECT COUNT(*) FROM ( + SELECT TOTAL(0) OVER (PARTITION BY t0.c0) FROM t0 + ) + WHERE ('1' IS NOT ('abcde' NOTNULL)); +} {1} + +# 2023-03-28 https://sqlite.org/forum/forumpost/dc3b92cfa0 (Song Liu) +# +reset_db +do_execsql_test 73.0 { + CREATE TABLE t1(a INT); + INSERT INTO t1(a) VALUES(1),(2),(4); + CREATE VIEW t2(b,c) AS SELECT * FROM t1 JOIN t1 A ORDER BY sum(0) OVER(PARTITION BY 0); + CREATE TRIGGER x1 INSTEAD OF UPDATE ON t2 BEGIN SELECT true; END; +} +do_execsql_test 73.1 { + SELECT * FROM t2; +} {1 1 1 2 1 4 2 1 2 2 2 4 4 1 4 2 4 4} +do_execsql_test 73.2 { + UPDATE t2 SET c=99 WHERE b=4 RETURNING *; +} {4 99 4 99 4 99} +do_execsql_test 73.3 { + SELECT *, nth_value(15,2) OVER() FROM t2, t1 WHERE b=4; +} { + 4 1 1 15 + 4 2 1 15 + 4 4 1 15 + 4 1 2 15 + 4 2 2 15 + 4 4 2 15 + 4 1 4 15 + 4 2 4 15 + 4 4 4 15 +} +do_execsql_test 73.4 { + UPDATE t2 SET c=nth_value(15,2) OVER() FROM (SELECT * FROM t1) WHERE b=4 RETURNING *; +} { + 4 15 + 4 15 + 4 15 + 4 15 + 4 15 + 4 15 + 4 15 + 4 15 + 4 15 +} +do_execsql_test 73.5 { + DROP TRIGGER x1; +} +do_catchsql_test 73.6 { + UPDATE t2 SET c=99 WHERE b=4 RETURNING *; +} {1 {cannot modify t2 because it is a view}} +do_catchsql_test 73.7 { + UPDATE t2 SET c=nth_value(15,2) OVER() FROM (SELECT * FROM t1) WHERE b=4 RETURNING *; +} {1 {cannot modify t2 because it is a view}} + +# 2023-03-28 https://sqlite.org/forum/forumpost/bad532820c +# +reset_db +do_execsql_test 74.0 { + CREATE TABLE t1 (a INT, b INT); + CREATE TABLE t2 (c INT, d INT); + CREATE INDEX idx ON t1(abs(a)); + INSERT INTO t1 VALUES(1,2),(3,4); + INSERT INTO t2 VALUES(5,6),(7,8); +} +do_execsql_test 74.1 { + SELECT ( + SELECT count( a ) FROM t2 LIMIT 1 + ) + FROM t1; +} {2} ;# Verified using PG 14.2 +do_execsql_test 74.2 { + SELECT ( + SELECT count( a+c ) FROM t2 LIMIT 1 + ) + FROM t1; +} {2 2} ;# verified on PG 14.2. Crashes PG 9.6! +do_execsql_test 74.3 { + SELECT ( + SELECT count( ( SELECT(sum(0) OVER(ORDER BY c, abs(a))) ) ) + FROM t2 GROUP BY c LIMIT 1 + ) + FROM t1; +} {1 1} ;# verified on PG 14.2 +do_execsql_test 74.4 { + /* Original test case reported in https://sqlite.org/forum/forumpost/bad532820c + CREATE TABLE v0 (c1); + CREATE INDEX i ON v0 (c1, c1=1); + SELECT 0 FROM v0 AS a1 + WHERE (SELECT count((SELECT(sum(0) OVER(PARTITION BY(c1), (a1.c1=1) )))) + FROM v0 + GROUP BY hex(0)) + AND a1.c1=0; +} {} + +# 2023-04-11 https://sqlite.org/forum/forumpost/6c5678e3da +# An ALWAYS() turns out to be sometimes false. +# +do_execsql_test 75.0 { + DROP TABLE t1; + CREATE TABLE t1(a INT, b INT); + CREATE INDEX t1x ON t1(a+b); +} +do_catchsql_test 75.1 { + SELECT count((SELECT count(a0.a+a0.b) ORDER BY sum(0) OVER (PARTITION BY 0))) + FROM t1 AS a0 JOIN t1 AS a1 + GROUP BY a1.a; +} {1 {misuse of aggregate: count()}} + +# 2023-04-13 https://sqlite.org/forum/forumpost/0d48347967 +reset_db +do_execsql_test 76.0 { + CREATE TABLE t1(a INT, b INT); + INSERT INTO t1(a,b) VALUES (111,222),(111,223),(118,229); + CREATE INDEX t1a ON t1(a); + CREATE TABLE t2(x INT); + INSERT INTO t2 VALUES (333),(444),(555); +} +do_execsql_test 76.1 { + SELECT c, (SELECT c + sum(1) OVER ()) AS "res" + FROM t2 LEFT JOIN (SELECT +a AS c FROM t1) AS v1 ON true + GROUP BY c + ORDER by c; +} {111 112 118 119} +# ^^^^^^^^^^^^^^^^^-- results verified against PG 14.2 + +do_execsql_test 76.2 { + CREATE TABLE t3(x); + CREATE TABLE t4(y); + INSERT INTO t3 VALUES(100), (200), (400); + INSERT INTO t4 VALUES(100), (300), (400); +} +do_execsql_test 76.3 { + SELECT (SELECT y+sum(0) OVER ()) FROM t3 LEFT JOIN t4 ON x=y; +} {100 {} 400} +do_execsql_test 76.4 { + SELECT (SELECT y+sum(0) OVER ()) FROM t3 LEFT JOIN t4 ON x=y GROUP BY x; +} {100 {} 400} +do_execsql_test 76.5 { + SELECT (SELECT max(y)+sum(0) OVER ()) FROM t3 LEFT JOIN t4 ON x=y GROUP BY x; +} {100 {} 400} + finish_test diff --git a/test/with1.test b/test/with1.test index 91cbc79524..7400a7adf3 100644 --- a/test/with1.test +++ b/test/with1.test @@ -1124,7 +1124,7 @@ do_execsql_test 24.1 { CREATE VIEW v1 AS SELECT max(a), min(b) FROM t1 GROUP BY c; } do_test 24.1 { - set program [db eval {EXPLAIN SELECT 1 FROM v1,v1,v1}] + set program [db eval {EXPLAIN SELECT * FROM v1 AS aa, v1 AS bb, v1 AS cc}] expr [lsearch $program OpenDup]>0 } {1} do_execsql_test 24.2 { diff --git a/test/with3.test b/test/with3.test index 85889453a0..650740dcc1 100644 --- a/test/with3.test +++ b/test/with3.test @@ -109,6 +109,7 @@ ifcapable analyze { | `--RECURSIVE STEP | `--SCAN cnt |--SCAN y1 + |--BLOOM FILTER ON cnt (i=?) `--SEARCH cnt USING AUTOMATIC COVERING INDEX (i=?) }] } diff --git a/test/with6.test b/test/with6.test index 91d64fa297..2a4bfc646d 100644 --- a/test/with6.test +++ b/test/with6.test @@ -363,9 +363,13 @@ do_eqp_test 400 { | |--SEARCH raw USING INDEX sqlite_autoindex_raw_1 (country=? AND date>? AND date=0; i=((int)aKWNext[i])-1){\n"); + printf(" for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){\n"); printf(" if( aKWLen[i]!=n ) continue;\n"); printf(" zKW = &zKWText[aKWOffset[i]];\n"); printf("#ifdef SQLITE_ASCII\n"); @@ -687,7 +687,7 @@ int main(int argc, char **argv){ printf(" if( j=SQLITE_N_KEYWORD ) return SQLITE_ERROR;\n"); + printf(" i++;\n"); printf(" *pzName = zKWText + aKWOffset[i];\n"); printf(" *pnName = aKWLen[i];\n"); printf(" return SQLITE_OK;\n"); diff --git a/tool/speed-check.sh b/tool/speed-check.sh index 4e070565e0..06cbd35745 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -158,6 +158,9 @@ while test "$1" != ""; do --fp) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp" ;; + --stmtscanstatus) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtscanstatus" + ;; -*) CC_OPTS="$CC_OPTS $1" ;;