diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 7869e638c5..2cfdd2a66c 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -776,7 +776,7 @@ static int fts3PendingTermsAdd( int iLangid, /* Language id to use */ const char *zText, /* Text of document to be inserted */ int iCol, /* Column into which text is being inserted */ - u32 *pnWord /* OUT: Number of tokens inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ ){ int rc; int iStart = 0; @@ -840,7 +840,7 @@ static int fts3PendingTermsAdd( } pModule->xClose(pCsr); - *pnWord = nWord; + *pnWord += nWord; return (rc==SQLITE_DONE ? SQLITE_OK : rc); } @@ -1044,11 +1044,13 @@ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ sqlite3_value *pRowid, /* The docid to be deleted */ - u32 *aSz /* Sizes of deleted document written here */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ ){ int rc; sqlite3_stmt *pSelect; + assert( *pbFound==0 ); if( *pRC ) return; rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); if( rc==SQLITE_OK ){ @@ -1066,6 +1068,7 @@ static void fts3DeleteTerms( *pRC = rc; return; } + *pbFound = 1; } rc = sqlite3_reset(pSelect); }else{ @@ -3290,7 +3293,7 @@ static int fts3DoRebuild(Fts3Table *p){ int iCol; int iLangid = langidFromSelect(p, pStmt); rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); - aSz[p->nColumn] = 0; + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); @@ -5194,28 +5197,32 @@ int sqlite3Fts3DeferToken( static int fts3DeleteByRowid( Fts3Table *p, sqlite3_value *pRowid, - int *pnDoc, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ u32 *aSzDel ){ - int isEmpty = 0; - int rc = fts3IsEmpty(p, pRowid, &isEmpty); - if( rc==SQLITE_OK ){ - if( isEmpty ){ - /* Deleting this row means the whole table is empty. In this case - ** delete the contents of all three tables and throw away any - ** data in the pendingTerms hash table. */ - rc = fts3DeleteAll(p, 1); - *pnDoc = *pnDoc - 1; - }else{ - fts3DeleteTerms(&rc, p, pRowid, aSzDel); - if( p->zContentTbl==0 ){ - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); - if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ + + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); }else{ - *pnDoc = *pnDoc - 1; - } - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } } } } @@ -5274,13 +5281,13 @@ int sqlite3Fts3UpdateMethod( } /* Allocate space to hold the change in document sizes */ - aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ){ + aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); + if( aSzDel==0 ){ rc = SQLITE_NOMEM; goto update_out; } - aSzDel = &aSzIns[p->nColumn+1]; - memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); /* If this is an INSERT operation, or an UPDATE that modifies the rowid ** value, then this operation requires constraint handling. @@ -5365,7 +5372,7 @@ int sqlite3Fts3UpdateMethod( } update_out: - sqlite3_free(aSzIns); + sqlite3_free(aSzDel); sqlite3Fts3SegmentsClose(p); return rc; } diff --git a/main.mk b/main.mk index a48999e6d8..61514a8ca5 100644 --- a/main.mk +++ b/main.mk @@ -258,6 +258,7 @@ TESTSRC = \ $(TOP)/src/test_schema.c \ $(TOP)/src/test_server.c \ $(TOP)/src/test_stat.c \ + $(TOP)/src/test_sqllog.c \ $(TOP)/src/test_superlock.c \ $(TOP)/src/test_syscall.c \ $(TOP)/src/test_tclvar.c \ diff --git a/manifest b/manifest index 6217c562fc..aba83b00bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\schanges\sfrom\strunk:\schiefly\sthe\souter/inner\sloop\squery\noptimizer\sscoring\senhancement\sand\sthe\sINSTR()\sfunction. -D 2012-11-10T01:27:59.154 +C Update\sthe\ssessions\sbranch\sto\sinclude\sthe\sSQLLOG\senhancement,\sthe\nSQLITE_IOERR_DELETE_NOENT\sfix,\sand\sa\sfix\sfor\sthe\snumber-of-documents\nbug\sin\sFTS4. +D 2012-11-27T21:56:28.635 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 82c41c0ed4cc94dd3cc7d498575b84c57c2c2384 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -72,7 +72,7 @@ F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 49e36e6ba59f79e6bd6a8bfe434570fe48d20559 F ext/fts3/fts3_unicode2.c a863f05f758af36777dffc2facc898bc73fec896 -F ext/fts3/fts3_write.c ba0bb0a91ca792fba5101bd82fa14d8a00a96365 +F ext/fts3/fts3_write.c a432433a706bd065e8bb0f8b3b33ce7cf9d7f21d F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -115,7 +115,7 @@ F ext/session/sqlite3session.h f374c9c4c96e08f67ac418871c29d423245c7673 F ext/session/test_session.c ea4dc9b4a1895c8e6bddcbfe3838d7eb57df2d99 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk faf97ab7bd203e56b5d93975c80abd32c814ddec +F main.mk d87518f07b121bd397117bf1f144f399388c7e06 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -146,7 +146,7 @@ F src/expr.c 3b25a95f3d309403940ba4a3212f197b8b6251d5 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c c82a04e7a92bb728f9ab972b76590403283be2af F src/func.c 1755cafdb8f2a291681f1ea55e0215518ebd6e52 -F src/global.c fb44b11e02e06c995e6ed6642509edd23599d584 +F src/global.c e59ecd2c553ad0d4bfbc84ca71231336f8993a7a F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -155,7 +155,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d -F src/main.c febaf66b42c36433b170e4704fef5b8b073a6d61 +F src/main.c e8e33e613698f3138031c037ce9d87a0030955c1 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -172,8 +172,8 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f0753566e1125d8b2eef6dd080b48ed91a83d424 -F src/os_win.c 43ec1285357e5d5d919cb0492eac775c58ad7d12 +F src/os_unix.c b75d9b0876ad3fde151843ad389b4c3dd727c662 +F src/os_win.c 6e55b48f793d0c0d0e086d3f1482a0882530eeeb F src/pager.c ed53fe75a269c1d67645fe079ea0f3f0ce6492d5 F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 @@ -188,10 +188,10 @@ F src/resolve.c 7b986a715ac281643309c29257bb58cfae7aa810 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 3a8baf4719f9723b4e0b43f2baa60692d0d921f8 F src/shell.c 24cd0aa74aff73ea08594629faead564c4c2a286 -F src/sqlite.h.in 6c9386f4be831a293bc7bac803f735e12a666ac1 +F src/sqlite.h.in 36d91c7ca79b9fb35831a0f097f4fcb653819a10 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h b071b5d387b9d256b1dcd2d0240fe50a74dbb898 +F src/sqliteInt.h 3a67c14ba5f17c33f2f9c551bb03c4c341de0bd4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -233,6 +233,7 @@ F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f F src/test_spellfix.c 76dd8d3111d2f5354c374f71fa23b752bd0b029c +F src/test_sqllog.c 7813b47021a6d4e39bb7b1b328a8893dc59885cb F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae @@ -252,7 +253,7 @@ F src/vdbe.c 640a7e140c7dc0465eff7e515252e434ca77286e F src/vdbe.h 1223e2548e0970cf96f573ff6b99f804a36ad683 F src/vdbeInt.h 2de43968dc47f1961d5bc76aa3cb68eacf433a7c F src/vdbeapi.c 58fdcd56109c05876f69c25d47a138ef370d3647 -F src/vdbeaux.c 5f53f96cda10e467ae308876157cad42572d9299 +F src/vdbeaux.c b8f2a0cb28401b7252bdbaba8988f53691909497 F src/vdbeblob.c 11248c6362389569764682eb0f59ce910f3cc381 F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74 F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 @@ -483,7 +484,7 @@ F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c -F test/fts3conf.test 8e65ea56f88ced6cdd2252bdddb1a8327ae5af7e +F test/fts3conf.test ee8500c86dd58ec075e8831a1e216a79989436de F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 @@ -956,7 +957,7 @@ F test/wal5.test f58ed4b8b542f71c7441da12fbd769d99b362437 F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 -F test/wal9.test b4eb5d27170c65ee9c8ff9c9e76babd902a13cfc +F test/wal9.test 48c40803faf6849515c81213697e9f3376835981 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test b9f68e39646375c2b877be906babcc15d38b4877 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 @@ -1037,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fce667f2d93a4ba65ccf1e748469576a3cd7ffcc 5a3b07f0f5dfae7eea870303f52f37d6a17f1da2 -R 2cae8c229d6424df7351cab19781b1e6 +P 2993ca20207f8dac02f58d01e31d68c84328356a f0843f885ab3337f83fe3b304aab80bb7b5dd0a3 +R 4a1915adbe009074546a4e2d976a205c U drh -Z 883d57cdfacddb67b9acee024d2c5aac +Z 7494d16bf0eedbea369965ccf52a8519 diff --git a/manifest.uuid b/manifest.uuid index 2c0d0dd237..e7e9d26c85 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2993ca20207f8dac02f58d01e31d68c84328356a \ No newline at end of file +ba8d08b67021a32fda069c18b7eb93523e6f0d1f \ No newline at end of file diff --git a/src/global.c b/src/global.c index dc86e1e081..f5da7e7f1f 100644 --- a/src/global.c +++ b/src/global.c @@ -175,6 +175,10 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* xLog */ 0, /* pLogArg */ 0, /* bLocaltimeFault */ +#ifdef SQLITE_ENABLE_SQLLOG + 0, /* xSqllog */ + 0 /* pSqllogArg */ +#endif }; diff --git a/src/main.c b/src/main.c index 5778f8780a..b8de245761 100644 --- a/src/main.c +++ b/src/main.c @@ -132,6 +132,13 @@ int sqlite3_initialize(void){ */ if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; +#ifdef SQLITE_ENABLE_SQLLOG + { + extern void sqlite3_init_sqllog(void); + sqlite3_init_sqllog(); + } +#endif + /* Make sure the mutex subsystem is initialized. If unable to ** initialize the mutex subsystem, return early with the error. ** If the system is so sick that we are unable to allocate a mutex, @@ -480,6 +487,15 @@ int sqlite3_config(int op, ...){ break; } +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + default: { rc = SQLITE_ERROR; break; @@ -819,6 +835,13 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ return SQLITE_BUSY; } +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + } +#endif + /* Convert the connection into a zombie and then close it. */ db->magic = SQLITE_MAGIC_ZOMBIE; @@ -2472,6 +2495,13 @@ opendb_out: db->magic = SQLITE_MAGIC_SICK; } *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif return sqlite3ApiExit(0, rc); } diff --git a/src/os_unix.c b/src/os_unix.c index ca62139430..a3a012126b 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -46,6 +46,13 @@ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ +/* Use posix_fallocate() if it is available +*/ +#if !defined(HAVE_POSIX_FALLOCATE) \ + && (_XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L) +# define HAVE_POSIX_FALLOCATE 1 +#endif + /* ** There are various methods for file locking used for concurrency ** control: @@ -4168,11 +4175,19 @@ static int unixShmMap( ** the requested memory region. */ if( !bExtend ) goto shmpage_out; +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + if( osFallocate(pShmNode->h, sStat.st_size, nByte)!=0 ){ + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "fallocate", + pShmNode->zFilename); + goto shmpage_out; + } +#else if( robust_ftruncate(pShmNode->h, nByte) ){ rc = unixLogError(SQLITE_IOERR_SHMSIZE, "ftruncate", pShmNode->zFilename); goto shmpage_out; } +#endif } } diff --git a/src/os_win.c b/src/os_win.c index 3c92b43d4e..6f49257705 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3888,14 +3888,24 @@ static int winDelete( &sAttrData) ){ attr = sAttrData.dwFileAttributes; }else{ - rc = SQLITE_OK; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } #else attr = osGetFileAttributesW(zConverted); #endif if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = SQLITE_OK; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ @@ -3917,7 +3927,12 @@ static int winDelete( do { attr = osGetFileAttributesA(zConverted); if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = SQLITE_OK; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ @@ -3935,7 +3950,7 @@ static int winDelete( } while(1); } #endif - if( rc ){ + if( rc && rc!=SQLITE_IOERR_DELETE_NOENT ){ rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, "winDelete", zFilename); }else{ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0522973f89..95fb0190b9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1593,6 +1593,22 @@ struct sqlite3_mem_methods { **
These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. ** +** +** [[SQLITE_CONFIG_SQLLOG]] +**
SQLITE_CONFIG_SQLLOG +**
This option is only available if sqlite is compiled with the +** SQLITE_ENABLE_SQLLOG pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. +** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1614,6 +1630,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 658b67ba4f..c3085ffd81 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2527,6 +2527,10 @@ struct Sqlite3Config { void (*xLog)(void*,int,const char*); /* Function for logging */ void *pLogArg; /* First argument to xLog() */ int bLocaltimeFault; /* True to fail localtime() calls */ +#ifdef SQLITE_ENABLE_SQLLOG + void(*xSqllog)(void*,sqlite3*,const char*, int); + void *pSqllogArg; +#endif }; /* diff --git a/src/test_sqllog.c b/src/test_sqllog.c new file mode 100644 index 0000000000..49569a39fb --- /dev/null +++ b/src/test_sqllog.c @@ -0,0 +1,472 @@ +/* +** 2012 November 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** OVERVIEW +** +** This file contains experimental code used to record data from live +** SQLite applications that may be useful for offline analysis. Specifically: +** +** 1) The initial contents of all database files opened by the +** application, and +** +** 2) All SQL statements executed by the application. +** +** USAGE +** +** To use this module, SQLite must be compiled with the SQLITE_ENABLE_SQLLOG +** pre-processor symbol defined and this file linked into the application +** somehow. +** +** At runtime, logging is enabled by setting environment variable +** SQLITE_SQLLOG_DIR to the name of a directory in which to store logged +** data. The directory must already exist. +** +** Usually, if the application opens the same database file more than once +** (either by attaching it or by using more than one database handle), only +** a single copy is made. This behaviour may be overridden (so that a +** separate copy is taken each time the database file is opened or attached) +** by setting the environment variable SQLITE_SQLLOG_REUSE_FILES to 0. +** +** OUTPUT: +** +** The SQLITE_SQLLOG_DIR is populated with three types of files: +** +** sqllog_N.db - Copies of database files. N may be any integer. +** +** sqllog_N.sql - A list of SQL statements executed by a single +** connection. N may be any integer. +** +** sqllog.idx - An index mapping from integer N to a database +** file name - indicating the full path of the +** database from which sqllog_N.db was copied. +** +** ERROR HANDLING: +** +** This module attempts to make a best effort to continue logging if an +** IO or other error is encountered. For example, if a log file cannot +** be opened logs are not collected for that connection, but other +** logging proceeds as expected. Errors are logged by calling sqlite3_log(). +*/ + +#include "sqlite3.h" +#include "stdio.h" +#include "stdlib.h" +#include "string.h" +#include "assert.h" + +#include "sys/types.h" +#include "unistd.h" +static int getProcessId(void){ +#if SQLITE_OS_WIN + return (int)_getpid(); +#else + return (int)getpid(); +#endif +} + + +#define ENVIRONMENT_VARIABLE1_NAME "SQLITE_SQLLOG_DIR" +#define ENVIRONMENT_VARIABLE2_NAME "SQLITE_SQLLOG_REUSE_FILES" + +/* Assume that all database and database file names are shorted than this. */ +#define SQLLOG_NAMESZ 512 + +/* Maximum number of simultaneous database connections the process may +** open (if any more are opened an error is logged using sqlite3_log() +** and processing is halted). +*/ +#define MAX_CONNECTIONS 256 + +struct SLConn { + int isErr; /* True if an error has occurred */ + sqlite3 *db; /* Connection handle */ + int iLog; /* First integer value used in file names */ + FILE *fd; /* File descriptor for log file */ +}; + +struct SLGlobal { + /* Protected by MUTEX_STATIC_MASTER */ + sqlite3_mutex *mutex; /* Recursive mutex */ + int nConn; /* Size of aConn[] array */ + + /* Protected by SLGlobal.mutex */ + int bReuse; /* True to avoid extra copies of db files */ + char zPrefix[SQLLOG_NAMESZ]; /* Prefix for all created files */ + char zIdx[SQLLOG_NAMESZ]; /* Full path to *.idx file */ + int iNextLog; /* Used to allocate file names */ + int iNextDb; /* Used to allocate database file names */ + int bRec; /* True if testSqllog() is called rec. */ + int iClock; /* Clock value */ + struct SLConn aConn[MAX_CONNECTIONS]; +} sqllogglobal; + +/* +** Return true if c is an ASCII whitespace character. +*/ +static int sqllog_isspace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\v' || c=='\f' || c=='\r'); +} + +/* +** The first argument points to a nul-terminated string containing an SQL +** command. Before returning, this function sets *pz to point to the start +** of the first token in this command, and *pn to the number of bytes in +** the token. This is used to check if the SQL command is an "ATTACH" or +** not. +*/ +static void sqllogTokenize(const char *z, const char **pz, int *pn){ + const char *p = z; + int n; + + /* Skip past any whitespace */ + while( sqllog_isspace(*p) ){ + p++; + } + + /* Figure out how long the first token is */ + *pz = p; + n = 0; + while( (p[n]>='a' && p[n]<='z') || (p[n]>='A' && p[n]<='Z') ) n++; + *pn = n; +} + +/* +** Check if the logs directory already contains a copy of database file +** zFile. If so, return a pointer to the full path of the copy. Otherwise, +** return NULL. +** +** If a non-NULL value is returned, then the caller must arrange to +** eventually free it using sqlite3_free(). +*/ +static char *sqllogFindFile(const char *zFile){ + char *zRet = 0; + FILE *fd = 0; + + /* Open the index file for reading */ + fd = fopen(sqllogglobal.zIdx, "r"); + if( fd==0 ){ + sqlite3_log(SQLITE_IOERR, "sqllogFindFile(): error in fopen()"); + return 0; + } + + /* Loop through each entry in the index file. If zFile is not NULL and the + ** entry is a match, then set zRet to point to the filename of the existing + ** copy and break out of the loop. */ + while( feof(fd)==0 ){ + char zLine[SQLLOG_NAMESZ*2+5]; + if( fgets(zLine, sizeof(zLine), fd) ){ + int n; + char *z; + + zLine[sizeof(zLine)-1] = '\0'; + z = zLine; + while( *z>='0' && *z<='9' ) z++; + while( *z==' ' ) z++; + + n = strlen(z); + while( n>0 && sqllog_isspace(z[n-1]) ) n--; + + if( n==strlen(zFile) && 0==memcmp(zFile, z, n) ){ + char zBuf[16]; + memset(zBuf, 0, sizeof(zBuf)); + z = zLine; + while( *z>='0' && *z<='9' ){ + zBuf[z-zLine] = *z; + z++; + } + zRet = sqlite3_mprintf("%s_%s.db", sqllogglobal.zPrefix, zBuf); + break; + } + } + } + + if( ferror(fd) ){ + sqlite3_log(SQLITE_IOERR, "sqllogFindFile(): error reading index file"); + } + + fclose(fd); + return zRet; +} + +static int sqllogFindAttached( + struct SLConn *p, /* Database connection */ + const char *zSearch, /* Name to search for (or NULL) */ + char *zName, /* OUT: Name of attached database */ + char *zFile /* OUT: Name of attached file */ +){ + sqlite3_stmt *pStmt; + int rc; + + /* The "PRAGMA database_list" command returns a list of databases in the + ** order that they were attached. So a newly attached database is + ** described by the last row returned. */ + assert( sqllogglobal.bRec==0 ); + sqllogglobal.bRec = 1; + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zVal1; int nVal1; + const char *zVal2; int nVal2; + + zVal1 = (const char*)sqlite3_column_text(pStmt, 1); + nVal1 = sqlite3_column_bytes(pStmt, 1); + memcpy(zName, zVal1, nVal1+1); + + zVal2 = (const char*)sqlite3_column_text(pStmt, 2); + nVal2 = sqlite3_column_bytes(pStmt, 2); + memcpy(zFile, zVal2, nVal2+1); + + if( zSearch && strlen(zSearch)==nVal1 + && 0==sqlite3_strnicmp(zSearch, zVal1, nVal1) + ){ + break; + } + } + rc = sqlite3_finalize(pStmt); + } + sqllogglobal.bRec = 0; + + if( rc!=SQLITE_OK ){ + sqlite3_log(rc, "sqllogFindAttached(): error in \"PRAGMA database_list\""); + } + return rc; +} + + +/* +** Parameter zSearch is the name of a database attached to the database +** connection associated with the first argument. This function creates +** a backup of this database in the logs directory. +** +** The name used for the backup file is automatically generated. Call +** it zFile. +** +** If the bLog parameter is true, then a statement of the following form +** is written to the log file associated with *p: +** +** ATTACH 'zFile' AS 'zName'; +** +** Otherwise, if bLog is false, a comment is added to the log file: +** +** -- Main database file is 'zFile' +** +** The SLGlobal.mutex mutex is always held when this function is called. +*/ +static void sqllogCopydb(struct SLConn *p, const char *zSearch, int bLog){ + char zName[SQLLOG_NAMESZ]; /* Attached database name */ + char zFile[SQLLOG_NAMESZ]; /* Database file name */ + char *zFree; + char *zInit = 0; + int rc; + + rc = sqllogFindAttached(p, zSearch, zName, zFile); + if( rc!=SQLITE_OK ) return; + + if( zFile[0]=='\0' ){ + zInit = sqlite3_mprintf(""); + }else{ + if( sqllogglobal.bReuse ){ + zInit = sqllogFindFile(zFile); + }else{ + zInit = 0; + } + if( zInit==0 ){ + int rc; + sqlite3 *copy = 0; + int iDb; + + /* Generate a file-name to use for the copy of this database */ + iDb = sqllogglobal.iNextDb++; + zInit = sqlite3_mprintf("%s_%d.db", sqllogglobal.zPrefix, iDb); + + /* Create the backup */ + assert( sqllogglobal.bRec==0 ); + sqllogglobal.bRec = 1; + rc = sqlite3_open(zInit, ©); + if( rc==SQLITE_OK ){ + sqlite3_backup *pBak; + sqlite3_exec(copy, "PRAGMA synchronous = 0", 0, 0, 0); + pBak = sqlite3_backup_init(copy, "main", p->db, zName); + if( pBak ){ + sqlite3_backup_step(pBak, -1); + rc = sqlite3_backup_finish(pBak); + }else{ + rc = sqlite3_errcode(copy); + } + sqlite3_close(copy); + } + sqllogglobal.bRec = 0; + + if( rc==SQLITE_OK ){ + /* Write an entry into the database index file */ + FILE *fd = fopen(sqllogglobal.zIdx, "a"); + if( fd ){ + fprintf(fd, "%d %s\n", iDb, zFile); + fclose(fd); + } + }else{ + sqlite3_log(rc, "sqllogCopydb(): error backing up database"); + } + } + } + + if( bLog ){ + zFree = sqlite3_mprintf("ATTACH '%q' AS '%q'; -- clock=%d\n", + zInit, zName, sqllogglobal.iClock++ + ); + }else{ + zFree = sqlite3_mprintf("-- Main database is '%q'\n", zInit); + } + fprintf(p->fd, "%s", zFree); + sqlite3_free(zFree); + + sqlite3_free(zInit); +} + +/* +** If it is not already open, open the log file for connection *p. +** +** The SLGlobal.mutex mutex is always held when this function is called. +*/ +static void sqllogOpenlog(struct SLConn *p){ + /* If the log file has not yet been opened, open it now. */ + if( p->fd==0 ){ + char *zLog; + + /* If it is still NULL, have global.zPrefix point to a copy of + ** environment variable $ENVIRONMENT_VARIABLE1_NAME. */ + if( sqllogglobal.zPrefix[0]==0 ){ + FILE *fd; + char *zVar = getenv(ENVIRONMENT_VARIABLE1_NAME); + if( zVar==0 || strlen(zVar)+10>=(sizeof(sqllogglobal.zPrefix)) ) return; + sprintf(sqllogglobal.zPrefix, "%s/sqllog_%d", zVar, getProcessId()); + sprintf(sqllogglobal.zIdx, "%s.idx", sqllogglobal.zPrefix); + if( getenv(ENVIRONMENT_VARIABLE2_NAME) ){ + sqllogglobal.bReuse = atoi(getenv(ENVIRONMENT_VARIABLE2_NAME)); + } + fd = fopen(sqllogglobal.zIdx, "w"); + if( fd ) fclose(fd); + } + + /* Open the log file */ + zLog = sqlite3_mprintf("%s_%d.sql", sqllogglobal.zPrefix, p->iLog); + p->fd = fopen(zLog, "w"); + sqlite3_free(zLog); + if( p->fd==0 ){ + sqlite3_log(SQLITE_IOERR, "sqllogOpenlog(): Failed to open log file"); + } + } +} + +/* +** This function is called if the SQLLOG callback is invoked to report +** execution of an SQL statement. Parameter p is the connection the statement +** was executed by and parameter zSql is the text of the statement itself. +*/ +static void testSqllogStmt(struct SLConn *p, const char *zSql){ + const char *zFirst; /* Pointer to first token in zSql */ + int nFirst; /* Size of token zFirst in bytes */ + + sqllogTokenize(zSql, &zFirst, &nFirst); + if( nFirst!=6 || 0!=sqlite3_strnicmp("ATTACH", zFirst, 6) ){ + /* Not an ATTACH statement. Write this directly to the log. */ + fprintf(p->fd, "%s; -- clock=%d\n", zSql, sqllogglobal.iClock++); + }else{ + /* This is an ATTACH statement. Copy the database. */ + sqllogCopydb(p, 0, 1); + } +} + +/* +** The SQLITE_CONFIG_SQLLOG callback registered by sqlite3_init_sqllog(). +*/ +static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int eType){ + struct SLConn *p; + sqlite3_mutex *master = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); + + assert( eType==0 || eType==1 || eType==2 ); + assert( (eType==2)==(zSql==0) ); + + /* This is a database open command. */ + if( eType==0 ){ + sqlite3_mutex_enter(master); + if( sqllogglobal.mutex==0 ){ + sqllogglobal.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE); + } + p = &sqllogglobal.aConn[sqllogglobal.nConn++]; + p->fd = 0; + p->db = db; + p->iLog = sqllogglobal.iNextLog++; + sqlite3_mutex_leave(master); + + /* Open the log and take a copy of the main database file */ + sqlite3_mutex_enter(sqllogglobal.mutex); + if( sqllogglobal.bRec==0 ){ + sqllogOpenlog(p); + if( p->fd ) sqllogCopydb(p, "main", 0); + } + sqlite3_mutex_leave(sqllogglobal.mutex); + } + + else{ + + int i; + for(i=0; idb==db ) break; + } + if( i==sqllogglobal.nConn ) return; + + /* A database handle close command */ + if( eType==2 ){ + sqlite3_mutex_enter(master); + if( p->fd ) fclose(p->fd); + p->db = 0; + p->fd = 0; + + sqllogglobal.nConn--; + if( sqllogglobal.nConn==0 ){ + sqlite3_mutex_free(sqllogglobal.mutex); + sqllogglobal.mutex = 0; + }else{ + int nShift = &sqllogglobal.aConn[sqllogglobal.nConn] - p; + if( nShift>0 ){ + memmove(p, &p[1], nShift*sizeof(struct SLConn)); + } + } + sqlite3_mutex_leave(master); + + /* An ordinary SQL command. */ + }else if( p->fd ){ + sqlite3_mutex_enter(sqllogglobal.mutex); + if( sqllogglobal.bRec==0 ){ + testSqllogStmt(p, zSql); + } + sqlite3_mutex_leave(sqllogglobal.mutex); + } + } +} + +/* +** This function is called either before sqlite3_initialized() or by it. +** It checks if the SQLITE_SQLLOG_DIR variable is defined, and if so +** registers an SQLITE_CONFIG_SQLLOG callback to record the applications +** database activity. +*/ +void sqlite3_init_sqllog(void){ + if( getenv(ENVIRONMENT_VARIABLE1_NAME) ){ + if( SQLITE_OK==sqlite3_config(SQLITE_CONFIG_SQLLOG, testSqllog, 0) ){ + memset(&sqllogglobal, 0, sizeof(sqllogglobal)); + sqllogglobal.bReuse = 1; + } + } +} diff --git a/src/vdbeaux.c b/src/vdbeaux.c index c37a0c5664..7170f982e4 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -53,7 +53,7 @@ Vdbe *sqlite3VdbeCreate(sqlite3 *db){ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){ assert( isPrepareV2==1 || isPrepareV2==0 ); if( p==0 ) return; -#ifdef SQLITE_OMIT_TRACE +#if defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_ENABLE_SQLLOG) if( !isPrepareV2 ) return; #endif assert( p->zSql==0 ); @@ -2327,6 +2327,27 @@ int sqlite3VdbeTransferError(Vdbe *p){ return rc; } +#ifdef SQLITE_ENABLE_SQLLOG +/* +** If an SQLITE_CONFIG_SQLLOG hook is registered and the VM has been run, +** invoke it. +*/ +static void vdbeInvokeSqllog(Vdbe *v){ + if( sqlite3GlobalConfig.xSqllog && v->rc==SQLITE_OK && v->zSql && v->pc>=0 ){ + char *zExpanded = sqlite3VdbeExpandSql(v, v->zSql); + assert( v->db->init.busy==0 ); + if( zExpanded ){ + sqlite3GlobalConfig.xSqllog( + sqlite3GlobalConfig.pSqllogArg, v->db, zExpanded, 1 + ); + sqlite3DbFree(v->db, zExpanded); + } + } +} +#else +# define vdbeInvokeSqllog(x) +#endif + /* ** Clean up a VDBE after execution but do not delete the VDBE just yet. ** Write any error messages into *pzErrMsg. Return the result code. @@ -2354,6 +2375,7 @@ int sqlite3VdbeReset(Vdbe *p){ ** instructions yet, leave the main database error information unchanged. */ if( p->pc>=0 ){ + vdbeInvokeSqllog(p); sqlite3VdbeTransferError(p); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; diff --git a/test/fts3conf.test b/test/fts3conf.test index ce410277ca..e91efbefbe 100644 --- a/test/fts3conf.test +++ b/test/fts3conf.test @@ -136,4 +136,46 @@ do_execsql_test 2.2.2 { COMMIT } do_execsql_test 2.2.3 { SELECT * FROM t1 } {{a b c} {a b c}} fts3_integrity 2.2.4 db t1 +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t3 USING fts4; + REPLACE INTO t3(docid, content) VALUES (1, 'one two'); + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one' +} {X'0100000002000000'} + +do_execsql_test 3.2 { + REPLACE INTO t3(docid, content) VALUES (2, 'one two three four'); + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'four' +} {X'0200000003000000'} + +do_execsql_test 3.3 { + REPLACE INTO t3(docid, content) VALUES (1, 'one two three four five six'); + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six' +} {X'0200000005000000'} + +do_execsql_test 3.4 { + UPDATE OR REPLACE t3 SET docid = 2 WHERE docid=1; + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six' +} {X'0100000006000000'} + +do_execsql_test 3.5 { + UPDATE OR REPLACE t3 SET docid = 3 WHERE docid=2; + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six' +} {X'0100000006000000'} + +do_execsql_test 3.6 { + REPLACE INTO t3(docid, content) VALUES (3, 'one two'); + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one' +} {X'0100000002000000'} + +do_execsql_test 3.7 { + REPLACE INTO t3(docid, content) VALUES (NULL, 'one two three four'); + REPLACE INTO t3(docid, content) VALUES (NULL, 'one two three four five six'); + SELECT docid FROM t3; +} {3 4 5} + +do_execsql_test 3.8 { + UPDATE OR REPLACE t3 SET docid = 5, content='three four' WHERE docid = 4; + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one' +} {X'0200000002000000'} + finish_test diff --git a/test/wal9.test b/test/wal9.test index a219e30884..f483eb615c 100644 --- a/test/wal9.test +++ b/test/wal9.test @@ -62,7 +62,7 @@ do_execsql_test 1.2 { # the *shm file is now more than one chunk (>32KiB). do_test 1.3 { file size test.db } {1024} do_test 1.4 { file size test.db-wal } {15421352} -do_test 1.5 { file size test.db-shm } {131072} +do_test 1.5 { expr {[file size test.db-shm]>32768} } {1} do_execsql_test 1.6 { PRAGMA wal_checkpoint } {0 14715 14715} @@ -87,4 +87,3 @@ do_test 1.7 { db2 close finish_test -