diff --git a/Makefile.in b/Makefile.in index 66eb903ba4..563f25860c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -190,7 +190,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo userauth.lo upsert.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ - vdbetrace.lo vdbetrace.lo \ + vdbetrace.lo vdbevtab.lo \ wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ window.lo utf.lo vtab.lo @@ -1078,6 +1078,7 @@ SHELL_SRC = \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/completion.c \ $(TOP)/ext/misc/sqlar.c \ + $(TOP)/ext/misc/uint.c \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ diff --git a/Makefile.msc b/Makefile.msc index 7ffdad1acb..b2d5862d90 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2188,6 +2188,7 @@ SHELL_SRC = \ $(TOP)\ext\misc\shathree.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\completion.c \ + $(TOP)\ext\misc\uint.c \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\sqlite3expert.h \ $(TOP)\ext\misc\memtrace.c \ diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index d03f3adf87..841d7448f9 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -962,6 +962,22 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ return zRet; } +/* +** Buffer z contains a positive integer value encoded as utf-8 text. +** Decode this value and store it in *pnOut, returning the number of bytes +** consumed. If an overflow error occurs return a negative value. +*/ +int sqlite3Fts3ReadInt(const char *z, int *pnOut){ + u64 iVal = 0; + int i; + for(i=0; z[i]>='0' && z[i]<='9'; i++){ + iVal = iVal*10 + (z[i] - '0'); + if( iVal>0x7FFFFFFF ) return -1; + } + *pnOut = (int)iVal; + return i; +} + /* ** This function interprets the string at (*pp) as a non-negative integer ** value. It reads the integer and sets *pnOut to the value read, then @@ -977,19 +993,17 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ */ static int fts3GobbleInt(const char **pp, int *pnOut){ const int MAX_NPREFIX = 10000000; - const char *p; /* Iterator pointer */ int nInt = 0; /* Output value */ - - for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ - nInt = nInt * 10 + (p[0] - '0'); - if( nInt>MAX_NPREFIX ){ - nInt = 0; - break; - } + int nByte; + nByte = sqlite3Fts3ReadInt(*pp, &nInt); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + } + if( nByte==0 ){ + return SQLITE_ERROR; } - if( p==*pp ) return SQLITE_ERROR; *pnOut = nInt; - *pp = p; + *pp += nByte; return SQLITE_OK; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 50370a9108..453afcebfd 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -591,6 +591,7 @@ int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); void sqlite3Fts3CreateStatTable(int*, Fts3Table*); int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); +int sqlite3Fts3ReadInt(const char *z, int *pnOut); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 5775fbca3c..e19137a03d 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -446,10 +446,7 @@ static int getNextNode( if( pKey->eType==FTSQUERY_NEAR ){ assert( nKey==4 ); if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ - nNear = 0; - for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ - nNear = nNear * 10 + (zInput[nKey] - '0'); - } + nKey += 1+sqlite3Fts3ReadInt(&zInput[nKey+1], &nNear); } } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index f5114eca42..b9acc47dc5 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1415,6 +1415,7 @@ static int fts3SegReaderNext( */ if( pReader->nDoclist > pReader->nNode-(pReader->aDoclist-pReader->aNode) || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) + || pReader->nDoclist==0 ){ return FTS_CORRUPT_VTAB; } @@ -3068,11 +3069,11 @@ static void fts3ReadEndBlockField( if( zText ){ int i; int iMul = 1; - i64 iVal = 0; + u64 iVal = 0; for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } - *piEndBlock = iVal; + *piEndBlock = (i64)iVal; while( zText[i]==' ' ) i++; iVal = 0; if( zText[i]=='-' ){ @@ -3082,7 +3083,7 @@ static void fts3ReadEndBlockField( for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } - *pnByte = (iVal * (i64)iMul); + *pnByte = ((i64)iVal * (i64)iMul); } } diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c new file mode 100644 index 0000000000..57f91f3c1a --- /dev/null +++ b/ext/misc/cksumvfs.c @@ -0,0 +1,778 @@ +/* +** 2020-04-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a VFS shim that writes a checksum on each page +** of an SQLite database file. When reading pages, the checksum is verified +** and an error is raised if the checksum is incorrect. +** +** COMPILING +** +** This extension requires SQLite 3.32.0 or later. It uses the +** sqlite3_database_file_object() interface which was added in +** version 3.32.0, so it will not link with an earlier version of +** SQLite. +** +** To build this extension as a separately loaded shared library or +** DLL, use compiler command-lines similar to the following: +** +** (linux) gcc -fPIC -shared cksumvfs.c -o cksumvfs.so +** (mac) clang -fPIC -dynamiclib cksumvfs.c -o cksumvfs.dylib +** (windows) cl cksumvfs.c -link -dll -out:cksumvfs.dll +** +** You may want to add additional compiler options, of course, +** according to the needs of your project. +** +** If you want to statically link this extension with your product, +** then compile it like any other C-language module but add the +** "-DSQLITE_CKSUMVFS_STATIC" option so that this module knows that +** it is being statically linked rather than dynamically linked +** +** LOADING +** +** To load this extension as a shared library, you first have to +** bring up a dummy SQLite database connection to use as the argument +** to the sqlite3_load_extension() API call. Then you invoke the +** sqlite3_load_extension() API and shutdown the dummy database +** connection. All subsequent database connections that are opened +** will include this extension. For example: +** +** sqlite3 *db; +** sqlite3_open(":memory:", &db); +** sqlite3_load_extention(db, "./cksumvfs"); +** sqlite3_close(db); +** +** If this extension is compiled with -DSQLITE_CKSUMVFS_STATIC and +** statically linked against the application, initialize it using +** a single API call as follows: +** +** sqlite3_cksumvfs_init(); +** +** Cksumvfs is a VFS Shim. When loaded, "cksmvfs" becomes the new +** default VFS and it uses the prior default VFS as the next VFS +** down in the stack. This is normally what you want. However, it +** complex situations where multiple VFS shims are being loaded, +** it might be important to ensure that cksumvfs is loaded in the +** correct order so that it sequences itself into the default VFS +** Shim stack in the right order. +** +** USING +** +** Open database connections using the sqlite3_open() or +** sqlite3_open_v2() interfaces, as normal. Ordinary database files +** (without a checksum) will operate normally. Databases with +** checksums will return an SQLITE_IOERR_DATA error if a page is +** encountered that contains an invalid checksum. +** +** Checksumming only works on databases that have a reserve-bytes +** value of exactly 8. The default value for reserve-bytes is 0. +** Hence, newly created database files will omit the checksum by +** default. To create a database that includes a checksum, change +** the reserve-bytes value to 8 by runing: +** +** int n = 8; +** sqlite3_file_control(db, 0, SQLITE_FCNTL_RESERVED_BYTES, &n); +** +** If you do this immediately after creating a new database file, +** before anything else has been written into the file, then that +** might be all that you need to do. Otherwise, the API call +** above should be followed by: +** +** sqlite3_exec(db, "VACUUM", 0, 0, 0); +** +** It never hurts to run the VACUUM, even if you don't need it. +** If the database is in WAL mode, you should shutdown and +** reopen all database connections before continuing. +** +** From the CLI, use the ".filectrl reserve_bytes 8" command, +** followed by "VACUUM;". +** +** Note that SQLite allows the number of reserve-bytes to be +** increased but not decreased. So if a database file already +** has a reserve-bytes value greater than 8, there is no way to +** activate checksumming on that database, other than to dump +** and restore the database file. Note also that other extensions +** might also make use of the reserve-bytes. Checksumming will +** be incompatible with those other extensions. +** +** VERIFICATION OF CHECKSUMS +** +** If any checksum is incorrect, the "PRAGMA quick_check" command +** will find it. To verify that checksums are actually enabled +** and running, use the following query: +** +** SELECT count(*), verify_checksum(data) +** FROM sqlite_dbpage +** GROUP BY 2; +** +** There are three possible outputs form the verify_checksum() +** function: 1, 0, and NULL. 1 is returned if the checksum is +** correct. 0 is returned if the checksum is incorrect. NULL +** is returned if the page is unreadable. If checksumming is +** enabled, the read will fail if the checksum is wrong, so the +** usual result from verify_checksum() on a bad checksum is NULL. +** +** If everything is OK, the query above should return a single +** row where the second column is 1. Any other result indicates +** either that there is a checksum error, or checksum validation +** is disabled. +** +** CONTROLLING CHECKSUM VERIFICATION +** +** The cksumvfs extension implements a new PRAGMA statement that can +** be used to disable, re-enable, or query the status of checksum +** verification: +** +** PRAGMA checksum_verification; -- query status +** PRAGMA checksum_verification=OFF; -- disable verification +** PRAGMA checksum_verification=ON; -- re-enable verification +** +** The "checksum_verification" pragma will return "1" (true) or "0" +** (false) if checksum verification is enabled or disabled, respectively. +** "Verification" in this context means the feature that causes +** SQLITE_IOERR_DATA errors if a checksum mismatch is detected while +** reading. Checksums are always kept up-to-date as long as the +** reserve-bytes value of the database is 8, regardless of the setting +** of this pragma. Checksum verification can be disabled (for example) +** to do forensic analysis of a database that has previously reported +** a checksum error. +** +** The "checksum_verification" pragma will always respond with "0" if +** the database file does not have a reserve-bytes value of 8. The +** pragma will return no rows at all if the cksumvfs extension is +** not loaded. +** +** IMPLEMENTATION NOTES +** +** The checksum is stored in the last 8 bytes of each page. This +** module only operates if the "bytes of reserved space on each page" +** value at offset 20 the SQLite database header is exactly 8. If +** the reserved-space value is not 8, this module is a no-op. +*/ +#ifdef SQLITE_CKSUMVFS_STATIC +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif +#include +#include + + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs CksmVfs; +typedef struct CksmFile CksmFile; + +/* +** Useful datatype abbreviations +*/ +#if !defined(SQLITE_CORE) + typedef unsigned char u8; + typedef unsigned int u32; +#endif + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((CksmFile*)(p))+1)) + +/* An open file */ +struct CksmFile { + sqlite3_file base; /* IO methods */ + const char *zFName; /* Original name of the file */ + char computeCksm; /* True to compute checksums. + ** Always true if reserve size is 8. */ + char verifyCksm; /* True to verify checksums */ + char isWal; /* True if processing a WAL file */ + CksmFile *pPartner; /* Ptr from WAL to main-db, or from main-db to WAL */ +}; + +/* +** Methods for CksmFile +*/ +static int cksmClose(sqlite3_file*); +static int cksmRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int cksmWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int cksmTruncate(sqlite3_file*, sqlite3_int64 size); +static int cksmSync(sqlite3_file*, int flags); +static int cksmFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int cksmLock(sqlite3_file*, int); +static int cksmUnlock(sqlite3_file*, int); +static int cksmCheckReservedLock(sqlite3_file*, int *pResOut); +static int cksmFileControl(sqlite3_file*, int op, void *pArg); +static int cksmSectorSize(sqlite3_file*); +static int cksmDeviceCharacteristics(sqlite3_file*); +static int cksmShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int cksmShmLock(sqlite3_file*, int offset, int n, int flags); +static void cksmShmBarrier(sqlite3_file*); +static int cksmShmUnmap(sqlite3_file*, int deleteFlag); +static int cksmFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int cksmUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for CksmVfs +*/ +static int cksmOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int cksmDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int cksmAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int cksmFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *cksmDlOpen(sqlite3_vfs*, const char *zFilename); +static void cksmDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void cksmDlClose(sqlite3_vfs*, void*); +static int cksmRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int cksmSleep(sqlite3_vfs*, int microseconds); +static int cksmCurrentTime(sqlite3_vfs*, double*); +static int cksmGetLastError(sqlite3_vfs*, int, char *); +static int cksmCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int cksmSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr cksmGetSystemCall(sqlite3_vfs*, const char *z); +static const char *cksmNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs cksm_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "cksmvfs", /* zName */ + 0, /* pAppData (set when registered) */ + cksmOpen, /* xOpen */ + cksmDelete, /* xDelete */ + cksmAccess, /* xAccess */ + cksmFullPathname, /* xFullPathname */ + cksmDlOpen, /* xDlOpen */ + cksmDlError, /* xDlError */ + cksmDlSym, /* xDlSym */ + cksmDlClose, /* xDlClose */ + cksmRandomness, /* xRandomness */ + cksmSleep, /* xSleep */ + cksmCurrentTime, /* xCurrentTime */ + cksmGetLastError, /* xGetLastError */ + cksmCurrentTimeInt64, /* xCurrentTimeInt64 */ + cksmSetSystemCall, /* xSetSystemCall */ + cksmGetSystemCall, /* xGetSystemCall */ + cksmNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods cksm_io_methods = { + 3, /* iVersion */ + cksmClose, /* xClose */ + cksmRead, /* xRead */ + cksmWrite, /* xWrite */ + cksmTruncate, /* xTruncate */ + cksmSync, /* xSync */ + cksmFileSize, /* xFileSize */ + cksmLock, /* xLock */ + cksmUnlock, /* xUnlock */ + cksmCheckReservedLock, /* xCheckReservedLock */ + cksmFileControl, /* xFileControl */ + cksmSectorSize, /* xSectorSize */ + cksmDeviceCharacteristics, /* xDeviceCharacteristics */ + cksmShmMap, /* xShmMap */ + cksmShmLock, /* xShmLock */ + cksmShmBarrier, /* xShmBarrier */ + cksmShmUnmap, /* xShmUnmap */ + cksmFetch, /* xFetch */ + cksmUnfetch /* xUnfetch */ +}; + +/* Do byte swapping on a unsigned 32-bit integer */ +#define BYTESWAP32(x) ( \ + (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ +) + +/* Compute a checksum on a buffer */ +static void cksmCompute( + u8 *a, /* Content to be checksummed */ + int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */ + u8 *aOut /* OUT: Final 8-byte checksum value output */ +){ + u32 s1 = 0, s2 = 0; + u32 *aData = (u32*)a; + u32 *aEnd = (u32*)&a[nByte]; + u32 x = 1; + + assert( nByte>=8 ); + assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); + + if( 1 == *(u8*)&x ){ + /* Little-endian */ + do { + s1 += *aData++ + s2; + s2 += *aData++ + s1; + }while( aData65536 || (nByte & (nByte-1))!=0 ) return; + cksmCompute(data, nByte-8, cksum); + sqlite3_result_int(context, memcmp(data+nByte-8,cksum,8)==0); +} + +/* +** Close a cksm-file. +*/ +static int cksmClose(sqlite3_file *pFile){ + CksmFile *p = (CksmFile *)pFile; + if( p->pPartner ){ + assert( p->pPartner->pPartner==p ); + p->pPartner->pPartner = 0; + p->pPartner = 0; + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Read data from a cksm-file. +*/ +static int cksmRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK ){ + if( iOfst==0 && iAmt>=100 && memcmp(zBuf,"SQLite format 3",16)==0 ){ + u8 *d = (u8*)zBuf; + char hasCorrectReserveSize = (d[20]==8); + if( hasCorrectReserveSize!=p->computeCksm ){ + p->computeCksm = p->verifyCksm = hasCorrectReserveSize; + } + } + /* Verify the checksum if (1) the size seems appropriate for a database + ** page, and if (2) checksum verification is enabled. But (3) do not + ** verify the checksums on a WAL page if the main database file + ** has subsequently turned off checksums. + */ + if( iAmt>=512 /* (1) */ + && p->verifyCksm /* (2) */ + && (!p->isWal || (p->pPartner!=0 && p->pPartner->verifyCksm)) /* (3) */ + ){ + u8 cksum[8]; + cksmCompute((u8*)zBuf, iAmt-8, cksum); + if( memcmp(zBuf+iAmt-8, cksum, 8)!=0 ){ + sqlite3_log(SQLITE_IOERR_DATA, + "checksum fault offset %lld of \"%s\"", + iOfst, p->zFName); + rc = SQLITE_IOERR_DATA; + } + } + } + return rc; +} + +/* +** Write data to a cksm-file. +*/ +static int cksmWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(pFile); + if( iOfst==0 && iAmt>=100 && memcmp(zBuf,"SQLite format 3",16)==0 ){ + u8 *d = (u8*)zBuf; + char hasCorrectReserveSize = (d[20]==8); + if( hasCorrectReserveSize!=p->computeCksm ){ + p->computeCksm = p->verifyCksm = hasCorrectReserveSize; + } + } + /* If the write size is appropriate for a database page and if + ** checksums where ever enabled, then it will be safe to compute + ** the checksums. The reserve byte size might have increased, but + ** it will never decrease. And because it cannot decrease, the + ** checksum will not overwrite anything. + */ + if( iAmt>=512 && p->computeCksm ){ + cksmCompute((u8*)zBuf, iAmt-8, ((u8*)zBuf)+iAmt-8); + } + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst); +} + +/* +** Truncate a cksm-file. +*/ +static int cksmTruncate(sqlite3_file *pFile, sqlite_int64 size){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xTruncate(pFile, size); +} + +/* +** Sync a cksm-file. +*/ +static int cksmSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of a cksm-file. +*/ +static int cksmFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(p); + return pFile->pMethods->xFileSize(pFile, pSize); +} + +/* +** Lock a cksm-file. +*/ +static int cksmLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock a cksm-file. +*/ +static int cksmUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on a cksm-file. +*/ +static int cksmCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on a cksm-file. +*/ +static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){ + int rc; + CksmFile *p = (CksmFile*)pFile; + pFile = ORIGFILE(pFile); + if( op==SQLITE_FCNTL_PRAGMA ){ + char **azArg = (char**)pArg; + assert( azArg[1]!=0 ); + if( sqlite3_stricmp(azArg[1],"checksum_verification")==0 ){ + char *zArg = azArg[2]; + if( zArg!=0 ){ + if( (zArg[0]>='1' && zArg[0]<='9') + || sqlite3_strlike("enable%",zArg,0)==0 + || sqlite3_stricmp("yes",zArg)==0 + || sqlite3_stricmp("on",zArg)==0 + ){ + p->verifyCksm = p->computeCksm; + }else{ + p->verifyCksm = 0; + } + } + azArg[0] = sqlite3_mprintf("%d",p->verifyCksm); + return SQLITE_OK; + }else if( p->computeCksm && azArg[2]!=0 + && sqlite3_stricmp(azArg[1], "page_size")==0 ){ + /* Do not allow page size changes on a checksum database */ + return SQLITE_OK; + } + } + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("cksm/%z", *(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for a cksm-file. +*/ +static int cksmSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by a cksm-file. +*/ +static int cksmDeviceCharacteristics(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xDeviceCharacteristics(pFile); +} + +/* Create a shared memory file mapping */ +static int cksmShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int cksmShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void cksmShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int cksmShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int cksmFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + CksmFile *p = (CksmFile *)pFile; + if( p->computeCksm ){ + *pp = 0; + return SQLITE_OK; + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xFetch(pFile, iOfst, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int cksmUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnfetch(pFile, iOfst, pPage); +} + +/* +** Open a cksm file handle. +*/ +static int cksmOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + CksmFile *p; + sqlite3_file *pSubFile; + sqlite3_vfs *pSubVfs; + int rc; + pSubVfs = ORIGVFS(pVfs); + if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ + return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + } + p = (CksmFile*)pFile; + memset(p, 0, sizeof(*p)); + pSubFile = ORIGFILE(pFile); + p->base.pMethods = &cksm_io_methods; + rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); + if( rc ) goto cksm_open_done; + if( flags & SQLITE_OPEN_WAL ){ + sqlite3_file *pDb = sqlite3_database_file_object(zName); + p->pPartner = (CksmFile*)pDb; + assert( p->pPartner->pPartner==0 ); + p->pPartner->pPartner = p; + p->isWal = 1; + p->computeCksm = p->pPartner->computeCksm; + }else{ + p->isWal = 0; + p->computeCksm = 0; + } + p->zFName = zName; +cksm_open_done: + if( rc ) pFile->pMethods = 0; + return rc; +} + +/* +** All other VFS methods are pass-thrus. +*/ +static int cksmDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} +static int cksmAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); +} +static int cksmFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); +} +static void *cksmDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} +static void cksmDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} +static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} +static void cksmDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} +static int cksmRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} +static int cksmSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} +static int cksmCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +static int cksmGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int cksmCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} +static int cksmSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr cksmGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *cksmNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} + +/* Register the verify_checksum() SQL function. +*/ +static int cksmRegisterFunc( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + if( db==0 ) return SQLITE_OK; + rc = sqlite3_create_function(db, "verify_checksum", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, cksmVerifyFunc, 0, 0); + return rc; +} + +/* +** Register the cksum VFS as the default VFS for the system. +** Also make arrangements to automatically register the "verify_checksum()" +** SQL function on each new database connection. +*/ +static int cksmRegisterVfs(void){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig; + if( sqlite3_vfs_find("cksum")!=0 ) return SQLITE_OK; + pOrig = sqlite3_vfs_find(0); + cksm_vfs.iVersion = pOrig->iVersion; + cksm_vfs.pAppData = pOrig; + cksm_vfs.szOsFile = pOrig->szOsFile + sizeof(CksmFile); + rc = sqlite3_vfs_register(&cksm_vfs, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))cksmRegisterFunc); + } + return rc; +} + +#if defined(SQLITE_CKSUMVFS_STATIC) +/* This variant of the initializer runs when the extension is +** statically linked. +*/ +int sqlite3_register_cksumvfs(const char *NotUsed){ + (void)NotUsed; + return cksmRegisterVfs(); +} +#endif /* defined(SQLITE_CKSUMVFS_STATIC */ + +#if !defined(SQLITE_CKSUMVFS_STATIC) +/* This variant of the initializer function is used when the +** extension is shared library to be loaded at run-time. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called by sqlite3_load_extension() when the +** extension is first loaded. +***/ +int sqlite3_cksumvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* not used */ + rc = cksmRegisterFunc(db, 0, 0); + if( rc==SQLITE_OK ){ + + } + if( rc==SQLITE_OK ){ + rc = cksmRegisterVfs(); + } + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} +#endif /* !defined(SQLITE_CKSUMVFS_STATIC) */ diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 1335229f9e..d977d41ddc 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -394,6 +394,7 @@ static int writeFile( if( mtime>=0 ){ #if defined(_WIN32) +#if !SQLITE_OS_WINRT /* Windows */ FILETIME lastAccess; FILETIME lastWrite; @@ -424,6 +425,7 @@ static int writeFile( }else{ return 1; } +#endif #elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ /* Recent unix */ struct timespec times[2]; diff --git a/ext/misc/uint.c b/ext/misc/uint.c new file mode 100644 index 0000000000..12982c2a20 --- /dev/null +++ b/ext/misc/uint.c @@ -0,0 +1,91 @@ +/* +** 2020-04-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements the UINT collating sequence. +** +** UINT works like BINARY for text, except that embedded strings +** of digits compare in numeric order. +** +** * Leading zeros are handled properly, in the sense that +** they do not mess of the maginitude comparison of embedded +** strings of digits. "x00123y" is equal to "x123y". +** +** * Only unsigned integers are recognized. Plus and minus +** signs are ignored. Decimal points and exponential notation +** are ignored. +** +** * Embedded integers can be of arbitrary length. Comparison +** is *not* limited integers that can be expressed as a +** 64-bit machine integer. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/* +** Compare text in lexicographic order, except strings of digits +** compare in numeric order. +*/ +static int uintCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + int i=0, j=0, x; + while( ipBt; - assert( nReserve>=-1 && nReserve<=255 ); + assert( nReserve>=-1 && nReserve<=254 ); sqlite3BtreeEnter(p); + if( nReserve>=0 ){ + pBt->nReserveWanted = nReserve + 1; + } if( pBt->btsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); return SQLITE_READONLY; @@ -2911,14 +2914,15 @@ int sqlite3BtreeGetReserveNoMutex(Btree *p){ ** are intentually left unused. This is the "reserved" space that is ** sometimes used by extensions. ** -** If SQLITE_HAS_MUTEX is defined then the number returned is the -** greater of the current reserved space and the maximum requested -** reserve space. +** The value returned is the larger of the current reserve size and +** the latest reserve size requested by SQLITE_FILECTRL_RESERVE_BYTES. +** The amount of reserve can only grow - never shrink. */ -int sqlite3BtreeGetOptimalReserve(Btree *p){ +int sqlite3BtreeGetRequestedReserve(Btree *p){ int n; sqlite3BtreeEnter(p); - n = sqlite3BtreeGetReserveNoMutex(p); + n = ((int)p->pBt->nReserveWanted) - 1; + if( n<0 ) n = sqlite3BtreeGetReserveNoMutex(p); sqlite3BtreeLeave(p); return n; } diff --git a/src/btree.h b/src/btree.h index 4bd41f7f37..680742a21d 100644 --- a/src/btree.h +++ b/src/btree.h @@ -74,7 +74,7 @@ int sqlite3BtreeGetPageSize(Btree*); int sqlite3BtreeMaxPageCount(Btree*,int); u32 sqlite3BtreeLastPage(Btree*); int sqlite3BtreeSecureDelete(Btree*,int); -int sqlite3BtreeGetOptimalReserve(Btree*); +int sqlite3BtreeGetRequestedReserve(Btree*); int sqlite3BtreeGetReserveNoMutex(Btree *p); int sqlite3BtreeSetAutoVacuum(Btree *, int); int sqlite3BtreeGetAutoVacuum(Btree *); diff --git a/src/btreeInt.h b/src/btreeInt.h index e5149d97ea..34b33096ba 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -417,6 +417,7 @@ struct BtShared { #endif u8 inTransaction; /* Transaction state */ u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ + u8 nReserveWanted; /* 1 more than desired number of extra bytes per page */ u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ diff --git a/src/expr.c b/src/expr.c index 7165e4fa14..23e00db2ee 100644 --- a/src/expr.c +++ b/src/expr.c @@ -45,7 +45,7 @@ char sqlite3TableColumnAffinity(Table *pTab, int iCol){ char sqlite3ExprAffinity(const Expr *pExpr){ int op; while( ExprHasProperty(pExpr, EP_Skip) ){ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW ); pExpr = pExpr->pLeft; assert( pExpr!=0 ); } @@ -112,7 +112,7 @@ Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ */ Expr *sqlite3ExprSkipCollate(Expr *pExpr){ while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW ); pExpr = pExpr->pLeft; } return pExpr; @@ -131,7 +131,7 @@ Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; }else{ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW ); pExpr = pExpr->pLeft; } } diff --git a/src/loadext.c b/src/loadext.c index 06297e6bb6..067c47c17f 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -477,8 +477,17 @@ static const sqlite3_api_routines sqlite3Apis = { /* Version 3.32.0 and later */ sqlite3_create_filename, sqlite3_free_filename, + sqlite3_database_file_object, }; +/* True if x is the directory separator character +*/ +#if SQLITE_OS_WIN +# define DirSep(X) ((X)=='/'||(X)=='\\') +#else +# define DirSep(X) ((X)=='/') +#endif + /* ** Attempt to load an SQLite extension library contained in the file ** zFile. The entry point is zProc. zProc may be 0 in which case a @@ -580,7 +589,7 @@ static int sqlite3LoadExtension( return SQLITE_NOMEM_BKPT; } memcpy(zAltEntry, "sqlite3_", 8); - for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} + for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} iFile++; if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ diff --git a/src/main.c b/src/main.c index 9c7fe3c137..7322e28279 100644 --- a/src/main.c +++ b/src/main.c @@ -3849,6 +3849,13 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ }else if( op==SQLITE_FCNTL_DATA_VERSION ){ *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESERVE_BYTES ){ + int iNew = *(int*)pArg; + *(int*)pArg = sqlite3BtreeGetRequestedReserve(pBtree); + if( iNew>=0 && iNew<=254 ){ + sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); + } + rc = SQLITE_OK; }else{ rc = sqlite3OsFileControl(fd, op, pArg); } @@ -4065,20 +4072,6 @@ int sqlite3_test_control(int op, ...){ break; } - /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) - ** - ** Set the nReserve size to N for the main database on the database - ** connection db. - */ - case SQLITE_TESTCTRL_RESERVE: { - sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); - sqlite3_mutex_leave(db->mutex); - break; - } - /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) ** ** Enable or disable various optimizations for testing purposes. The diff --git a/src/malloc.c b/src/malloc.c index a19d8bdfb3..45f3efff9d 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -111,7 +111,7 @@ sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ } mem0.alarmThreshold = n; nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - mem0.nearlyFull = (n>0 && n<=nUsed); + AtomicStore(&mem0.nearlyFull, n>0 && n<=nUsed); sqlite3_mutex_leave(mem0.mutex); excess = sqlite3_memory_used() - n; if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); @@ -179,7 +179,7 @@ int sqlite3MallocInit(void){ ** sqlite3_soft_heap_limit(). */ int sqlite3HeapNearlyFull(void){ - return mem0.nearlyFull; + return AtomicLoad(&mem0.nearlyFull); } /* @@ -243,7 +243,7 @@ static void mallocWithAlarm(int n, void **pp){ if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ - mem0.nearlyFull = 1; + AtomicStore(&mem0.nearlyFull, 1); sqlite3MallocAlarm(nFull); if( mem0.hardLimit ){ nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); @@ -253,7 +253,7 @@ static void mallocWithAlarm(int n, void **pp){ } } }else{ - mem0.nearlyFull = 0; + AtomicStore(&mem0.nearlyFull, 0); } } p = sqlite3GlobalConfig.m.xMalloc(nFull); diff --git a/src/os_unix.c b/src/os_unix.c index a4dd2f906c..56e53929ea 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3685,7 +3685,7 @@ static int openDirectory(const char *zFilename, int *pFd){ if( zDirname[0]!='/' ) zDirname[0] = '.'; zDirname[1] = 0; } - fd = robust_open(zDirname, O_RDONLY|O_BINARY|O_NOFOLLOW, 0); + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); if( fd>=0 ){ OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); } diff --git a/src/pager.c b/src/pager.c index 5d2225590e..0d08a2dfaa 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2536,9 +2536,12 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ /* One of the journals pointed to by the master journal exists. ** Open it and check if it points at the master journal. If ** so, return without deleting the master journal file. + ** NB: zJournal is really a MAIN_JOURNAL. But call it a + ** MASTER_JOURNAL here so that the VFS will not send the zJournal + ** name into sqlite3_database_file_object(). */ int c; - int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL); + int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); rc = sqlite3OsOpen(pVfs, zJournal, pJournal, flags, 0); if( rc!=SQLITE_OK ){ goto delmaster_out; @@ -4742,6 +4745,7 @@ int sqlite3PagerOpen( ** Database file handle (pVfs->szOsFile bytes) ** Sub-journal file handle (journalFileSize bytes) ** Main journal file handle (journalFileSize bytes) + ** Ptr back to the Pager (sizeof(Pager*) bytes) ** \0\0\0\0 database prefix (4 bytes) ** Database file name (nPathname+1 bytes) ** URI query parameters (nUriByte bytes) @@ -4781,6 +4785,7 @@ int sqlite3PagerOpen( ROUND8(pcacheSize) + /* PCache object */ ROUND8(pVfs->szOsFile) + /* The main db file */ journalFileSize * 2 + /* The two journal files */ + sizeof(pPager) + /* Space to hold a pointer */ 4 + /* Database prefix */ nPathname + 1 + /* database filename */ nUriByte + /* query parameters */ @@ -4801,6 +4806,7 @@ int sqlite3PagerOpen( pPager->sjfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; pPager->jfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); + memcpy(pPtr, &pPager, sizeof(pPager)); pPtr += sizeof(pPager); /* Fill in the Pager.zFilename and pPager.zQueryParam fields */ pPtr += 4; /* Skip zero prefix */ @@ -5001,6 +5007,19 @@ act_like_temp_file: return SQLITE_OK; } +/* +** Return the sqlite3_file for the main database given the name +** of the corresonding WAL or Journal name as passed into +** xOpen. +*/ +sqlite3_file *sqlite3_database_file_object(const char *zName){ + Pager *pPager; + while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ + zName--; + } + pPager = *(Pager**)(zName - 4 - sizeof(Pager*)); + return pPager->fd; +} /* diff --git a/src/select.c b/src/select.c index 76b9827e8c..4b7ba37f9f 100644 --- a/src/select.c +++ b/src/select.c @@ -3498,6 +3498,7 @@ static Expr *substExpr( ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; ifNullRow.iTable = pSubst->iNewTable; + ifNullRow.flags = EP_Skip; pCopy = &ifNullRow; } testcase( ExprHasProperty(pCopy, EP_Subquery) ); diff --git a/src/shell.c.in b/src/shell.c.in index a35e863a71..f65caaa8ac 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -17,6 +17,14 @@ #define _CRT_SECURE_NO_WARNINGS #endif +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + /* ** Warning pragmas copied from msvc.h in the core. */ @@ -129,22 +137,26 @@ typedef unsigned char u8; #if defined(_WIN32) || defined(WIN32) -# include -# include -# define isatty(h) _isatty(h) -# ifndef access -# define access(f,m) _access((f),(m)) +# if SQLITE_OS_WINRT +# define SQLITE_OMIT_POPEN 1 +# else +# include +# include +# define isatty(h) _isatty(h) +# ifndef access +# define access(f,m) _access((f),(m)) +# endif +# ifndef unlink +# define unlink _unlink +# endif +# ifndef strdup +# define strdup _strdup +# endif +# undef popen +# define popen _popen +# undef pclose +# define pclose _pclose # endif -# ifndef unlink -# define unlink _unlink -# endif -# ifndef strdup -# define strdup _strdup -# endif -# undef popen -# define popen _popen -# undef pclose -# define pclose _pclose #else /* Make sure isatty() has a prototype. */ extern int isatty(int); @@ -173,6 +185,9 @@ typedef unsigned char u8; #define ToLower(X) (char)tolower((unsigned char)X) #if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT +#include +#endif #include /* string conversion routines only needed on Win32 */ @@ -188,7 +203,7 @@ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); ** rendering quoted strings that contain \n characters). The following ** routines take care of that. */ -#if defined(_WIN32) || defined(WIN32) +#if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT static void setBinaryMode(FILE *file, int isOutput){ if( isOutput ) fflush(file); _setmode(_fileno(file), _O_BINARY); @@ -292,6 +307,7 @@ static int hasTimer(void){ if( getProcessTimesAddr ){ return 1; } else { +#if !SQLITE_OS_WINRT /* GetProcessTimes() isn't supported in WIN95 and some other Windows ** versions. See if the version we are running on has it, and if it ** does, save off a pointer to it and the current process handle. @@ -308,6 +324,7 @@ static int hasTimer(void){ FreeLibrary(hinstLib); } } +#endif } return 0; } @@ -993,6 +1010,7 @@ INCLUDE ../ext/misc/fileio.c INCLUDE ../ext/misc/completion.c INCLUDE ../ext/misc/appendvfs.c INCLUDE ../ext/misc/memtrace.c +INCLUDE ../ext/misc/uint.c #ifdef SQLITE_HAVE_ZLIB INCLUDE ../ext/misc/zipfile.c INCLUDE ../ext/misc/sqlar.c @@ -1089,6 +1107,7 @@ struct ShellState { unsigned mxProgress; /* Maximum progress callbacks before failing */ unsigned flgProgress; /* Flags for the progress callback */ unsigned shellFlgs; /* Various flags */ + unsigned priorShFlgs; /* Saved copy of flags */ sqlite3_int64 szMax; /* --maxsize argument to .open */ char *zDestTable; /* Name of destination table when MODE_Insert */ char *zTempFile; /* Temporary file that might need deleting */ @@ -1389,11 +1408,13 @@ edit_func_end: */ static void outputModePush(ShellState *p){ p->modePrior = p->mode; + p->priorShFlgs = p->shellFlgs; memcpy(p->colSepPrior, p->colSeparator, sizeof(p->colSeparator)); memcpy(p->rowSepPrior, p->rowSeparator, sizeof(p->rowSeparator)); } static void outputModePop(ShellState *p){ p->mode = p->modePrior; + p->shellFlgs = p->priorShFlgs; memcpy(p->colSeparator, p->colSepPrior, sizeof(p->colSeparator)); memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator)); } @@ -3562,11 +3583,13 @@ static const char *(azHelp[]) = { #endif " trigger Like \"full\" but also show trigger bytecode", ".excel Display the output of next command in spreadsheet", + " --bom Put a UTF8 byte-order mark on intermediate file", ".exit ?CODE? Exit this program with return-code CODE", ".expert EXPERIMENTAL. Suggest indexes for queries", ".explain ?on|off|auto? Change the EXPLAIN formatting mode. Default: auto", ".filectrl CMD ... Run various sqlite3_file_control() operations", - " Run \".filectrl\" with no arguments for details", + " --schema SCHEMA Use SCHEMA instead of \"main\"", + " --help Show CMD details", ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", ".headers on|off Turn display of headers on or off", ".help ?-all? ?PATTERN? Show help text for PATTERN", @@ -3613,11 +3636,11 @@ static const char *(azHelp[]) = { " tabs Tab-separated values", " tcl TCL list elements", ".nullvalue STRING Use STRING in place of NULL values", - ".once (-e|-x|FILE) Output for the next SQL command only to FILE", + ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", " If FILE begins with '|' then open as a pipe", - " Other options:", - " -e Invoke system text editor", - " -x Open in a spreadsheet", + " --bom Put a UTF8 byte-order mark at the beginning", + " -e Send output to the system text editor", + " -x Send output as CSV to a spreadsheet (same as \".excel\")", #ifdef SQLITE_DEBUG ".oom [--repeat M] [N] Simulate an OOM error on the N-th allocation", #endif @@ -3634,7 +3657,11 @@ static const char *(azHelp[]) = { " --readonly Open FILE readonly", " --zip FILE is a ZIP archive", ".output ?FILE? Send output to FILE or stdout if FILE is omitted", - " If FILE begins with '|' then open it as a pipe.", + " If FILE begins with '|' then open it as a pipe.", + " Options:", + " --bom Prefix output with a UTF8 byte-order mark", + " -e Send output to the system text editor", + " -x Send output as CSV to a spreadsheet", ".parameter CMD ... Manage SQL parameter bindings", " clear Erase all bindings", " init Initialize the TEMP table that holds bindings", @@ -3754,6 +3781,7 @@ static int showHelp(FILE *out, const char *zPattern){ || zPattern[0]=='0' || strcmp(zPattern,"-a")==0 || strcmp(zPattern,"-all")==0 + || strcmp(zPattern,"--all")==0 ){ /* Show all commands, but only one line per command */ if( zPattern==0 ) zPattern = ""; @@ -4235,6 +4263,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); + sqlite3_uint_init(p->db, 0, 0); #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) sqlite3_dbdata_init(p->db, 0, 0); #endif @@ -4954,11 +4983,15 @@ static void output_reset(ShellState *p){ zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); if( system(zCmd) ){ utf8_printf(stderr, "Failed: [%s]\n", zCmd); + }else{ + /* Give the start/open/xdg-open command some time to get + ** going before we continue, and potential delete the + ** p->zTempFile data file out from under it */ + sqlite3_sleep(2000); } sqlite3_free(zCmd); outputModePop(p); p->doXdgOpen = 0; - sqlite3_sleep(100); } #endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ } @@ -5034,12 +5067,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", -1, &pStmt, 0); if( rc ){ - if( !sqlite3_compileoption_used("ENABLE_DBPAGE_VTAB") ){ - utf8_printf(stderr, "the \".dbinfo\" command requires the " - "-DSQLITE_ENABLE_DBPAGE_VTAB compile-time options\n"); - }else{ - utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); - } + utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); return 1; } @@ -5248,9 +5276,21 @@ static void newTempFile(ShellState *p, const char *zSuffix){ sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile); } if( p->zTempFile==0 ){ + /* If p->db is an in-memory database then the TEMPFILENAME file-control + ** will not work and we will need to fallback to guessing */ + char *zTemp; sqlite3_uint64 r; sqlite3_randomness(sizeof(r), &r); - p->zTempFile = sqlite3_mprintf("temp%llx.%s", r, zSuffix); + zTemp = getenv("TEMP"); + if( zTemp==0 ) zTemp = getenv("TMP"); + if( zTemp==0 ){ +#ifdef _WIN32 + zTemp = "\\tmp"; +#else + zTemp = "/tmp"; +#endif + } + p->zTempFile = sqlite3_mprintf("%s/temp%llx.%s", zTemp, r, zSuffix); }else{ p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix); } @@ -7445,6 +7485,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { "tempfilename", SQLITE_FCNTL_TEMPFILENAME, "" }, { "has_moved", SQLITE_FCNTL_HAS_MOVED, "" }, { "lock_timeout", SQLITE_FCNTL_LOCK_TIMEOUT, "MILLISEC" }, + { "reserve_bytes", SQLITE_FCNTL_RESERVE_BYTES, "[N]" }, }; int filectrl = -1; int iCtrl = -1; @@ -7452,10 +7493,21 @@ static int do_meta_command(char *zLine, ShellState *p){ int isOk = 0; /* 0: usage 1: %lld 2: no-result */ int n2, i; const char *zCmd = 0; + const char *zSchema = 0; open_db(p, 0); zCmd = nArg>=2 ? azArg[1] : "help"; + if( zCmd[0]=='-' + && (strcmp(zCmd,"--schema")==0 || strcmp(zCmd,"-schema")==0) + && nArg>=4 + ){ + zSchema = azArg[2]; + for(i=3; idb, 0, SQLITE_FCNTL_SIZE_LIMIT, &iRes); + sqlite3_file_control(p->db, zSchema, SQLITE_FCNTL_SIZE_LIMIT, &iRes); isOk = 1; break; } @@ -7506,7 +7558,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int x; if( nArg!=3 ) break; x = (int)integerValue(azArg[2]); - sqlite3_file_control(p->db, 0, filectrl, &x); + sqlite3_file_control(p->db, zSchema, filectrl, &x); isOk = 2; break; } @@ -7515,7 +7567,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int x; if( nArg!=2 && nArg!=3 ) break; x = nArg==3 ? booleanValue(azArg[2]) : -1; - sqlite3_file_control(p->db, 0, filectrl, &x); + sqlite3_file_control(p->db, zSchema, filectrl, &x); iRes = x; isOk = 1; break; @@ -7523,7 +7575,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_FCNTL_HAS_MOVED: { int x; if( nArg!=2 ) break; - sqlite3_file_control(p->db, 0, filectrl, &x); + sqlite3_file_control(p->db, zSchema, filectrl, &x); iRes = x; isOk = 1; break; @@ -7531,7 +7583,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_FCNTL_TEMPFILENAME: { char *z = 0; if( nArg!=2 ) break; - sqlite3_file_control(p->db, 0, filectrl, &z); + sqlite3_file_control(p->db, zSchema, filectrl, &z); if( z ){ utf8_printf(p->out, "%s\n", z); sqlite3_free(z); @@ -7539,6 +7591,18 @@ static int do_meta_command(char *zLine, ShellState *p){ isOk = 2; break; } + case SQLITE_FCNTL_RESERVE_BYTES: { + int x; + if( nArg>=3 ){ + x = atoi(azArg[2]); + sqlite3_file_control(p->db, zSchema, filectrl, &x); + } + x = -1; + sqlite3_file_control(p->db, zSchema, filectrl, &x); + utf8_printf(p->out,"%d\n", x); + isOk = 2; + break; + } } } if( isOk==0 && iCtrl>=0 ){ @@ -8272,42 +8336,66 @@ static int do_meta_command(char *zLine, ShellState *p){ && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) ){ - const char *zFile = nArg>=2 ? azArg[1] : "stdout"; + const char *zFile = 0; int bTxtMode = 0; - if( azArg[0][0]=='e' ){ - /* Transform the ".excel" command into ".once -x" */ - nArg = 2; - azArg[0] = "once"; - zFile = azArg[1] = "-x"; - n = 4; + int i; + int eMode = 0; + int bBOM = 0; + int bOnce = 0; /* 0: .output, 1: .once, 2: .excel */ + + if( c=='e' ){ + eMode = 'x'; + bOnce = 2; + }else if( strncmp(azArg[0],"once",n)==0 ){ + bOnce = 1; } - if( nArg>2 ){ - utf8_printf(stderr, "Usage: .%s [-e|-x|FILE]\n", azArg[0]); - rc = 1; - goto meta_command_exit; - } - if( n>1 && strncmp(azArg[0], "once", n)==0 ){ - if( nArg<2 ){ - raw_printf(stderr, "Usage: .once (-e|-x|FILE)\n"); + for(i=1; iout, "ERROR: unknown option: \"%s\". Usage:\n", + azArg[i]); + showHelp(p->out, azArg[0]); + rc = 1; + goto meta_command_exit; + } + }else if( zFile==0 ){ + zFile = z; + }else{ + utf8_printf(p->out,"ERROR: extra parameter: \"%s\". Usage:\n", + azArg[i]); + showHelp(p->out, azArg[0]); rc = 1; goto meta_command_exit; } + } + if( zFile==0 ) zFile = "stdout"; + if( bOnce ){ p->outCount = 2; }else{ p->outCount = 0; } output_reset(p); - if( zFile[0]=='-' && zFile[1]=='-' ) zFile++; #ifndef SQLITE_NOHAVE_SYSTEM - if( strcmp(zFile, "-e")==0 || strcmp(zFile, "-x")==0 ){ + if( eMode=='e' || eMode=='x' ){ p->doXdgOpen = 1; outputModePush(p); - if( zFile[1]=='x' ){ + if( eMode=='x' ){ + /* spreadsheet mode. Output as CSV. */ newTempFile(p, "csv"); + ShellClearFlag(p, SHFLG_Echo); p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); }else{ + /* text editor mode */ newTempFile(p, "txt"); bTxtMode = 1; } @@ -8326,6 +8414,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->out = stdout; rc = 1; }else{ + if( bBOM ) fprintf(p->out,"\357\273\277"); sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); } #endif @@ -8338,6 +8427,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->out = stdout; rc = 1; } else { + if( bBOM ) fprintf(p->out,"\357\273\277"); sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); } } @@ -9401,7 +9491,6 @@ static int do_meta_command(char *zLine, ShellState *p){ { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE, "" }, { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE, "" }, { "prng_seed", SQLITE_TESTCTRL_PRNG_SEED, "SEED ?db?" }, - { "reserve", SQLITE_TESTCTRL_RESERVE, "BYTES-OF-RESERVE"}, }; int testctrl = -1; int iCtrl = -1; @@ -9454,7 +9543,6 @@ static int do_meta_command(char *zLine, ShellState *p){ /* sqlite3_test_control(int, db, int) */ case SQLITE_TESTCTRL_OPTIMIZATIONS: - case SQLITE_TESTCTRL_RESERVE: if( nArg==3 ){ int opt = (int)strtol(azArg[2], 0, 0); rc2 = sqlite3_test_control(testctrl, p->db, opt); @@ -10226,14 +10314,18 @@ static void main_init(ShellState *data) { */ #ifdef _WIN32 static void printBold(const char *zText){ +#if !SQLITE_OS_WINRT HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo; GetConsoleScreenBufferInfo(out, &defaultScreenInfo); SetConsoleTextAttribute(out, FOREGROUND_RED|FOREGROUND_INTENSITY ); +#endif printf("%s", zText); +#if !SQLITE_OS_WINRT SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); +#endif } #else static void printBold(const char *zText){ @@ -10301,7 +10393,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ fgetc(stdin); }else{ #if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT + __debugbreak(); +#else DebugBreak(); +#endif #elif defined(SIGTRAP) raise(SIGTRAP); #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 86c3223801..f65576a15d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -299,26 +299,22 @@ typedef sqlite_uint64 sqlite3_uint64; ** the [sqlite3] object is successfully destroyed and all associated ** resources are deallocated. ** -** ^If the database connection is associated with unfinalized prepared -** statements or unfinished sqlite3_backup objects then sqlite3_close() -** will leave the database connection open and return [SQLITE_BUSY]. -** ^If sqlite3_close_v2() is called with unfinalized prepared statements -** and/or unfinished sqlite3_backups, then the database connection becomes -** an unusable "zombie" which will automatically be deallocated when the -** last prepared statement is finalized or the last sqlite3_backup is -** finished. The sqlite3_close_v2() interface is intended for use with -** host languages that are garbage collected, and where the order in which -** destructors are called is arbitrary. -** -** Applications should [sqlite3_finalize | finalize] all [prepared statements], -** [sqlite3_blob_close | close] all [BLOB handles], and +** Ideally, applications should [sqlite3_finalize | finalize] all +** [prepared statements], [sqlite3_blob_close | close] all [BLOB handles], and ** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated -** with the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close_v2() is called on a [database connection] that still has -** outstanding [prepared statements], [BLOB handles], and/or -** [sqlite3_backup] objects then it returns [SQLITE_OK] and the deallocation -** of resources is deferred until all [prepared statements], [BLOB handles], -** and [sqlite3_backup] objects are also destroyed. +** with the [sqlite3] object prior to attempting to close the object. +** ^If the database connection is associated with unfinalized prepared +** statements, BLOB handlers, and/or unfinished sqlite3_backup objects then +** sqlite3_close() will leave the database connection open and return +** [SQLITE_BUSY]. ^If sqlite3_close_v2() is called with unfinalized prepared +** statements, unclosed BLOB handlers, and/or unfinished sqlite3_backups, +** it returns [SQLITE_OK] regardless, but instead of deallocating the database +** connection immediately, it marks the database connection as an unusable +** "zombie" and makes arrangements to automatically deallocate the database +** connection after all prepared statements are finalized, all BLOB handles +** are closed, and all backups have finished. The sqlite3_close_v2() interface +** is intended for use with host languages that are garbage collected, and +** where the order in which destructors are called is arbitrary. ** ** ^If an [sqlite3] object is destroyed while a transaction is open, ** the transaction is automatically rolled back. @@ -507,6 +503,7 @@ int sqlite3_exec( #define SQLITE_IOERR_BEGIN_ATOMIC (SQLITE_IOERR | (29<<8)) #define SQLITE_IOERR_COMMIT_ATOMIC (SQLITE_IOERR | (30<<8)) #define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) +#define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -1157,6 +1154,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_DATA_VERSION 35 #define SQLITE_FCNTL_SIZE_LIMIT 36 #define SQLITE_FCNTL_CKPT_DONE 37 +#define SQLITE_FCNTL_RESERVE_BYTES 38 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -3535,8 +3533,19 @@ int sqlite3_open_v2( ** that check if a database file was a URI that contained a specific query ** parameter, and if so obtains the value of that query parameter. ** -** If F is the database filename pointer passed into the xOpen() method of -** a VFS implementation or it is the return value of [sqlite3_db_filename()] +** The first parameter to these interfaces (hereafter referred to +** as F) must be one of: +**
    +**
  • A database filename pointer created by the SQLite core and +** passed into the xOpen() method of a VFS implemention, or +**
  • A filename obtained from [sqlite3_db_filename()], or +**
  • A new filename constructed using [sqlite3_create_filename()]. +**
+** If the F parameter is not one of the above, then the behavior is +** undefined and probably undesirable. Older versions of SQLite were +** more tolerant of invalid F parameters than newer versions. +** +** If F is a suitable filename (as described in the previous paragraph) ** and if P is the name of the query parameter, then ** sqlite3_uri_parameter(F,P) returns the value of the P ** parameter if it exists or a NULL pointer if P does not appear as a @@ -3619,6 +3628,25 @@ const char *sqlite3_filename_database(const char*); const char *sqlite3_filename_journal(const char*); const char *sqlite3_filename_wal(const char*); +/* +** CAPI3REF: Database File Corresponding To A Journal +** +** ^If X is the name of a rollback or WAL-mode journal file that is +** passed into the xOpen method of [sqlite3_vfs], then +** sqlite3_database_file_object(X) returns a pointer to the [sqlite3_file] +** object that represents the main database file. +** +** This routine is intended for use in custom [VFS] implementations +** only. It is not a general-purpose interface. +** The argument sqlite3_file_object(X) must be a filename pointer that +** has been passed into [sqlite3_vfs].xOpen method where the +** flags parameter to xOpen contains one of the bits +** [SQLITE_OPEN_MAIN_JOURNAL] or [SQLITE_OPEN_WAL]. Any other use +** of this routine results in undefined and probably undesirable +** behavior. +*/ +sqlite3_file *sqlite3_database_file_object(const char*); + /* ** CAPI3REF: Create and Destroy VFS Filenames ** @@ -3653,7 +3681,7 @@ const char *sqlite3_filename_wal(const char*); ** ** The sqlite3_free_filename(Y) routine releases a memory allocation ** previously obtained from sqlite3_create_filename(). Invoking -** sqlite3_free_filename(Y) is a NULL pointer is a harmless no-op. +** sqlite3_free_filename(Y) where Y is a NULL pointer is a harmless no-op. ** ** If the Y parameter to sqlite3_free_filename(Y) is anything other ** than a NULL pointer or a pointer previously acquired from @@ -4260,6 +4288,24 @@ typedef struct sqlite3_context sqlite3_context; ** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() ** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter ** is ignored and the end result is the same as sqlite3_bind_null(). +** ^If the third parameter to sqlite3_bind_text() is not NULL, then +** it should be a pointer to well-formed UTF8 text. +** ^If the third parameter to sqlite3_bind_text16() is not NULL, then +** it should be a pointer to well-formed UTF16 text. +** ^If the third parameter to sqlite3_bind_text64() is not NULL, then +** it should be a pointer to a well-formed unicode string that is +** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 +** otherwise. +** +** [[byte-order determination rules]] ^The byte-order of +** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) +** found in first character, which is removed, or in the absence of a BOM +** the byte order is the native byte order of the host +** machine for sqlite3_bind_text16() or the byte order specified in +** the 6th parameter for sqlite3_bind_text64().)^ +** ^If UTF16 input text contains invalid unicode +** characters, then SQLite might change those invalid characters +** into the unicode replacement character: U+FFFD. ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the @@ -4273,7 +4319,7 @@ typedef struct sqlite3_context sqlite3_context; ** or sqlite3_bind_text16() or sqlite3_bind_text64() then ** that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL -** terminated. If any NUL characters occur at byte offsets less than +** terminated. If any NUL characters occurs at byte offsets less than ** the value of the fourth parameter then the resulting string value will ** contain embedded NULs. The result of expressions involving strings ** with embedded NULs is undefined. @@ -5598,8 +5644,9 @@ typedef void (*sqlite3_destructor_type)(void*); ** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() ** as the text of an error message. ^SQLite interprets the error ** message string from sqlite3_result_error() as UTF-8. ^SQLite -** interprets the string from sqlite3_result_error16() as UTF-16 in native -** byte order. ^If the third parameter to sqlite3_result_error() +** interprets the string from sqlite3_result_error16() as UTF-16 using +** the same [byte-order determination rules] as [sqlite3_bind_text16()]. +** ^If the third parameter to sqlite3_result_error() ** or sqlite3_result_error16() is negative then SQLite takes as the error ** message all text up through the first zero character. ** ^If the third parameter to sqlite3_result_error() or @@ -5667,6 +5714,25 @@ typedef void (*sqlite3_destructor_type)(void*); ** then SQLite makes a copy of the result into space obtained ** from [sqlite3_malloc()] before it returns. ** +** ^For the sqlite3_result_text16(), sqlite3_result_text16le(), and +** sqlite3_result_text16be() routines, and for sqlite3_result_text64() +** when the encoding is not UTF8, if the input UTF16 begins with a +** byte-order mark (BOM, U+FEFF) then the BOM is removed from the +** string and the rest of the string is interpreted according to the +** byte-order specified by the BOM. ^The byte-order specified by +** the BOM at the beginning of the text overrides the byte-order +** specified by the interface procedure. ^So, for example, if +** sqlite3_result_text16le() is invoked with text that begins +** with bytes 0xfe, 0xff (a big-endian byte-order mark) then the +** first two bytes of input are skipped and the remaining input +** is interpreted as UTF16BE text. +** +** ^For UTF16 input text to the sqlite3_result_text16(), +** sqlite3_result_text16be(), sqlite3_result_text16le(), and +** sqlite3_result_text64() routines, if the text contains invalid +** UTF16 characters, the invalid characters might be converted +** into the unicode replacement character, U+FFFD. +** ** ^The sqlite3_result_value() interface sets the result of ** the application-defined function to be a copy of the ** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The @@ -7614,7 +7680,7 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_PENDING_BYTE 11 #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 -#define SQLITE_TESTCTRL_RESERVE 14 +#define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index a7c76a6784..78c19a0d10 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -334,6 +334,7 @@ struct sqlite3_api_routines { char *(*create_filename)(const char*,const char*,const char*, int,const char**); void (*free_filename)(char*); + sqlite3_file *(*database_file_object)(const char*); }; /* @@ -637,6 +638,7 @@ typedef int (*sqlite3_loadext_entry)( /* Version 3.32.0 and later */ #define sqlite3_create_filename sqlite3_api->create_filename #define sqlite3_free_filename sqlite3_api->free_filename +#define sqlite3_database_file_object sqlite3_api->database_file_object #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/src/vacuum.c b/src/vacuum.c index ce9db649d2..5c230fbe14 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -233,7 +233,7 @@ SQLITE_NOINLINE int sqlite3RunVacuum( } db->mDbFlags |= DBFLAG_VacuumInto; } - nRes = sqlite3BtreeGetOptimalReserve(pMain); + nRes = sqlite3BtreeGetRequestedReserve(pMain); sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test index b11d19c7f1..89853833cb 100644 --- a/test/fts3corrupt4.test +++ b/test/fts3corrupt4.test @@ -5834,4 +5834,17 @@ do_catchsql_test 36.1 { INSERT INTO f(f) VALUES ('merge=59,59'); } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 37.0 { + CREATE VIRTUAL TABLE f USING fts3(a,b); + INSERT INTO f_segdir VALUES (28,0,0,0,'0 0',x'00'); + INSERT INTO f_segdir VALUES (0,241,0,0,'0 0',x'0001000030310000f1'); +} + +do_catchsql_test 37.1 { + INSERT INTO f VALUES (0,x'00'); +} {1 {database disk image is malformed}} + finish_test diff --git a/test/fts3misc.test b/test/fts3misc.test index 9becba9af7..a1bec42432 100644 --- a/test/fts3misc.test +++ b/test/fts3misc.test @@ -315,4 +315,12 @@ do_catchsql_test 10.1 { INSERT INTO f(f) VALUES ('merge=69,59'); } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE xyz USING fts3(); +} +do_execsql_test 11.1 { + SELECT * FROM xyz WHERE xyz MATCH 'a NEAR/4294836224 a'; +} + finish_test diff --git a/test/fts4aa.test b/test/fts4aa.test index 112d60ab4b..0c6c0b972f 100644 --- a/test/fts4aa.test +++ b/test/fts4aa.test @@ -234,13 +234,13 @@ if {$tcl_platform(byteOrder)=="littleEndian"} { } else { set res {X'0000000200000000000000000000000E0000000E00000001000000010000000100000001'} } -do_execsql_test fts4aa-6.10 { +do_catchsql_test fts4aa-6.10 { CREATE VIRTUAL TABLE f USING fts4(); INSERT INTO f_segdir VALUES (77,91,0,0,'255 77',x'0001308000004d5c4ddddddd4d4d7b4d4d4d614d8019ff4d05000001204d4d2e4d6e4d4d4d4b4d6c4d004d4d4d4d4d4d3d000000004d5d4d4d645d4d004d4d4d4d4d4d4d4d4d454d6910004d05ffff054d646c4d004d5d4d4d4d4d3d000000004d4d4d4d4d4d4d4d4d4d4d69624d4d4d04004d4d4d4d4d604d4ce1404d554d45'); INSERT INTO f_segdir VALUES (77,108,0,0,'255 77',x'0001310000fa64004d4d4d3c5d4d654d4d4d614d8000ff4d05000001204d4d2e4d6e4d4d4dff4d4d4d4d4d4d00104d4d4d4d000000004d4d4d0400311d4d4d4d4d4d4d4d4d4d684d6910004d05ffff054d4d6c4d004d4d4d4d4d4d3d000000004d4d4d4d644d4d4d4d4d4d69624d4d4d03ed4d4d4d4d4d604d4ce1404d550080'); INSERT INTO f_stat VALUES (0,x'80808080100000000064004d4d4d3c4d4d654d4d4d614d8000ff4df6ff1a00204d4d2e4d6e4d4d4d104d4d4d4d4d4d00104d4d4d4d4d4d69574d4d4d000031044d4d4d3e4d4d4c4d05004d6910'); SELECT quote(matchinfo(f,'pnax')) from f where f match '0 1'; -} $res +} {1 {database disk image is malformed}} # 2019-11-18 Detect infinite loop in fts3SelectLeaf() db close diff --git a/test/join2.test b/test/join2.test index bfcecda29b..82d597c584 100644 --- a/test/join2.test +++ b/test/join2.test @@ -293,5 +293,36 @@ do_execsql_test 8.1 { WHERE (t1.c0 BETWEEN 0 AND 0) > ('' AND t0.c0); } +#------------------------------------------------------------------------- +# Ticket [45f4bf4eb]. +# +reset_db +do_execsql_test 9.0 { + CREATE TABLE t0(c0 INT); + CREATE VIEW v0(c0) AS SELECT CAST(t0.c0 AS INTEGER) FROM t0; + INSERT INTO t0(c0) VALUES (0); +} + +do_execsql_test 9.1 { + SELECT typeof(c0), c0 FROM v0 WHERE c0>='0' +} {integer 0} + +do_execsql_test 9.2 { + SELECT * FROM t0, v0 WHERE v0.c0 >= '0'; +} {0 0} + +do_execsql_test 9.3 { + SELECT * FROM t0 LEFT JOIN v0 WHERE v0.c0 >= '0'; +} {0 0} + +do_execsql_test 9.4 { + SELECT * FROM t0 LEFT JOIN v0 ON v0.c0 >= '0'; +} {0 0} + +do_execsql_test 9.5 { + SELECT * FROM t0 LEFT JOIN v0 ON v0.c0 >= '0' WHERE TRUE + UNION SELECT 0,0 WHERE 0; +} {0 0} + finish_test diff --git a/test/shell1.test b/test/shell1.test index bbe2ad765e..c142ea7241 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -493,7 +493,14 @@ do_test shell1-3.15.2 { do_test shell1-3.15.3 { # too many arguments catchcmd "test.db" ".output FOO BAD" -} {1 {Usage: .output [-e|-x|FILE]}} +} {1 {ERROR: extra parameter: "BAD". Usage: +.output ?FILE? Send output to FILE or stdout if FILE is omitted + If FILE begins with '|' then open it as a pipe. + Options: + --bom Prefix output with a UTF8 byte-order mark + -e Send output to the system text editor + -x Send output as CSV to a spreadsheet +child process exited abnormally}} # .output stdout Send output to the screen do_test shell1-3.16.1 { @@ -502,7 +509,14 @@ do_test shell1-3.16.1 { do_test shell1-3.16.2 { # too many arguments catchcmd "test.db" ".output stdout BAD" -} {1 {Usage: .output [-e|-x|FILE]}} +} {1 {ERROR: extra parameter: "BAD". Usage: +.output ?FILE? Send output to FILE or stdout if FILE is omitted + If FILE begins with '|' then open it as a pipe. + Options: + --bom Prefix output with a UTF8 byte-order mark + -e Send output to the system text editor + -x Send output as CSV to a spreadsheet +child process exited abnormally}} # .prompt MAIN CONTINUE Replace the standard prompts do_test shell1-3.17.1 { diff --git a/test/whereL.test b/test/whereL.test index 0f577e06eb..fbb424e919 100644 --- a/test/whereL.test +++ b/test/whereL.test @@ -157,4 +157,38 @@ do_execsql_test 600 { WHERE x='good' AND y='good'; } {good good} +# 2020-04-24: Another test case for the previous (1dcb4d44964846ad) +# ticket. The test case comes from +# https://stackoverflow.com/questions/61399253/sqlite3-different-result-in-console-compared-to-python-script/ +# Output verified against postgresql. +# +do_execsql_test 610 { + CREATE TABLE tableA( + ID int, + RunYearMonth int + ); + INSERT INTO tableA VALUES(1,202003),(2,202003),(3,202003),(4,202004), + (5,202004),(6,202004),(7,202004),(8,202004); + CREATE TABLE tableB ( + ID int, + RunYearMonth int + ); + INSERT INTO tableB VALUES(1,202004),(2,202004),(3,202004),(4,202004), + (5,202004); + SELECT * + FROM ( + SELECT * + FROM tableA + WHERE RunYearMonth = 202004 + ) AS A + INNER JOIN ( + SELECT * + FROM tableB + WHERE RunYearMonth = 202004 + ) AS B + ON A.ID = B.ID + AND A.RunYearMonth = B.RunYearMonth; +} {4 202004 4 202004 5 202004 5 202004} + + finish_test diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl index 3e08d3ffa7..3b33f8868e 100644 --- a/tool/spaceanal.tcl +++ b/tool/spaceanal.tcl @@ -587,6 +587,9 @@ set free_percent2 [percent $free_pgcnt2 $file_pgcnt] set file_pgcnt2 [expr {$inuse_pgcnt+$free_pgcnt2+$av_pgcnt}] +# Account for the lockbyte page +if {$file_pgcnt2*$pageSize>1073742335} {incr file_pgcnt2} + set ntable [db eval {SELECT count(*)+1 FROM sqlite_master WHERE type='table'}] set nindex [db eval {SELECT count(*) FROM sqlite_master WHERE type='index'}] set sql {SELECT count(*) FROM sqlite_master WHERE name LIKE 'sqlite_autoindex%'}