From 96d7b510e9aeb26a8655d920f3e50926c5922bf0 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Feb 2012 20:43:05 +0000 Subject: [PATCH 01/51] Fix a warning coming from the Solaris Studio compiler. FossilOrigin-Name: 33294bbd1724665832464b33f865a29dc82b90f6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fb04f1bdd8..d2559bc518 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\sability\sto\shave\smultiple\svalues\sin\sthe\sVALUES\sclause\sof\san\nINSERT\sstatement\swhen\sSQLITE_OMIT_COMPOUND_SELECT\sis\sused. -D 2012-02-10T17:38:58.524 +C Fix\sa\swarning\scoming\sfrom\sthe\sSolaris\sStudio\scompiler. +D 2012-02-10T20:43:05.055 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -137,7 +137,7 @@ F src/delete.c 51d32f0a9c880663e54ce309f52e40c325d5e112 F src/expr.c 00675123e0beec98f999aa4594d2cbe1fec33c1b F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 657212460bf5cfd3ae607d12ea62092844c227b5 -F src/func.c d7925f33a8ce2207f86dea5cfb1a4379413312fe +F src/func.c 482fb61e26d1c0f983e788276d3700eee26442df F src/global.c 4cfdca5cb0edd33c4d021baec4ede958cb2c793b F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 1a16db0bca717a2582a48332d81854ca90d6d49b -R 4ae3522fd6eccbfc1737e8f5c9c63cf7 +P 92131195d0c24c0116992db51ed5d8316626ba57 +R adab601e293e6166c9f1fe94f2757bd1 U drh -Z 485b3d0d156b49609a8dc8c25c8298f5 +Z 57c4345b7c5ce6bd28783854ba285555 diff --git a/manifest.uuid b/manifest.uuid index 866f339ebe..e13aeafdfc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92131195d0c24c0116992db51ed5d8316626ba57 \ No newline at end of file +33294bbd1724665832464b33f865a29dc82b90f6 \ No newline at end of file diff --git a/src/func.c b/src/func.c index b7d0810686..385582923d 100644 --- a/src/func.c +++ b/src/func.c @@ -416,7 +416,7 @@ static void randomFunc( ** 2s complement of that positive value. The end result can ** therefore be no less than -9223372036854775807. */ - r = -(r ^ (((sqlite3_int64)1)<<63)); + r = -(r ^ (((sqlite3_uint64)1)<<63)); } sqlite3_result_int64(context, r); } From c96d1e79a80c115b5a0c2238ef332c9dcfd354d5 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Feb 2012 18:51:34 +0000 Subject: [PATCH 02/51] Remove a redundant test from the shared-memory logic in os_unix.c. FossilOrigin-Name: 31142ca795005bf664f34000591e6572c72652f2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d2559bc518..4c3f43e124 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\swarning\scoming\sfrom\sthe\sSolaris\sStudio\scompiler. -D 2012-02-10T20:43:05.055 +C Remove\sa\sredundant\stest\sfrom\sthe\sshared-memory\slogic\sin\sos_unix.c. +D 2012-02-11T18:51:34.899 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c d509b369ed376c77bc547961844a105d3907e4fa +F src/os_unix.c 35fb4bc9bc1acf2fb67d4c3b78d1ab471e22a0fd F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 92131195d0c24c0116992db51ed5d8316626ba57 -R adab601e293e6166c9f1fe94f2757bd1 +P 33294bbd1724665832464b33f865a29dc82b90f6 +R 00b08584e9c041156461ca40087c2e4e U drh -Z 57c4345b7c5ce6bd28783854ba285555 +Z c9bd46f5e0e3af1639554e403a310b26 diff --git a/manifest.uuid b/manifest.uuid index e13aeafdfc..b10ed1b83c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -33294bbd1724665832464b33f865a29dc82b90f6 \ No newline at end of file +31142ca795005bf664f34000591e6572c72652f2 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 5fad7aa2ce..b717a08d72 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3901,10 +3901,8 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777)); if( pShmNode->h<0 ){ - if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; - } + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); + goto shm_open_err; } /* Check to see if another process is holding the dead-man switch. From ac7c3ac150b00d2d4450894b873e0615601cb728 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Feb 2012 19:23:48 +0000 Subject: [PATCH 03/51] When creating journal files (including -wal and -shm files) try to set the ownership to be the same as the original database. This will prevent root from locking out the original owner of the file. FossilOrigin-Name: 1254dffe4071656a783cd000b1dd40c975ac18cb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 29 +++++++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4c3f43e124..bf76b978d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sredundant\stest\sfrom\sthe\sshared-memory\slogic\sin\sos_unix.c. -D 2012-02-11T18:51:34.899 +C When\screating\sjournal\sfiles\s(including\s-wal\sand\s-shm\sfiles)\stry\sto\sset\sthe\nownership\sto\sbe\sthe\ssame\sas\sthe\soriginal\sdatabase.\s\sThis\swill\sprevent\sroot\nfrom\slocking\sout\sthe\soriginal\sowner\sof\sthe\sfile. +D 2012-02-11T19:23:48.068 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 35fb4bc9bc1acf2fb67d4c3b78d1ab471e22a0fd +F src/os_unix.c f7e7b3e4f6922e3b07250a22c81da766ac2cc8fa F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 33294bbd1724665832464b33f865a29dc82b90f6 -R 00b08584e9c041156461ca40087c2e4e +P 31142ca795005bf664f34000591e6572c72652f2 +R bf3c1343906df15f83435e1a674bd11e U drh -Z c9bd46f5e0e3af1639554e403a310b26 +Z 10acee19c2f6805ba89a4f170956f4c5 diff --git a/manifest.uuid b/manifest.uuid index b10ed1b83c..be561debed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -31142ca795005bf664f34000591e6572c72652f2 \ No newline at end of file +1254dffe4071656a783cd000b1dd40c975ac18cb \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index b717a08d72..5e4ab8d248 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3904,6 +3904,13 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); goto shm_open_err; } + + /* If this process is running as root, make sure that the SHM file + ** is owned by the same user that owns the original database. Otherwise, + ** the original owner will not be able to connect. If this process is + ** not root, the following fchown() will fail, but we don't care. + */ + fchown(pShmNode->h, sStat.st_uid, sStat.st_gid); /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. @@ -4896,10 +4903,14 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ static int findCreateFileMode( const char *zPath, /* Path of file (possibly) being created */ int flags, /* Flags passed as 4th argument to xOpen() */ - mode_t *pMode /* OUT: Permissions to open file with */ + mode_t *pMode, /* OUT: Permissions to open file with */ + uid_t *pUid, /* OUT: uid to set on the file */ + gid_t *pGid /* OUT: gid to set on the file */ ){ int rc = SQLITE_OK; /* Return Code */ *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; + *pUid = 0; + *pGid = 0; if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ char zDb[MAX_PATHNAME+1]; /* Database file path */ int nDb; /* Number of valid bytes in zDb */ @@ -4933,6 +4944,8 @@ static int findCreateFileMode( if( 0==osStat(zDb, &sStat) ){ *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; }else{ rc = SQLITE_IOERR_FSTAT; } @@ -5079,7 +5092,9 @@ static int unixOpen( if( fd<0 ){ mode_t openMode; /* Permissions to create file with */ - rc = findCreateFileMode(zName, flags, &openMode); + uid_t uid; /* Userid for the file */ + gid_t gid; /* Groupid for the file */ + rc = findCreateFileMode(zName, flags, &openMode, &uid, &gid); if( rc!=SQLITE_OK ){ assert( !p->pUnused ); assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); @@ -5100,6 +5115,16 @@ static int unixOpen( rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); goto open_finished; } + + /* If this process is running as root and if creating a new rollback + ** journal or WAL file, set the ownership of the journal or WAL to be + ** the same as the original database. If we are not running as root, + ** then the fchown() call will fail, but that's ok - there is nothing + ** we can do about it so just ignore the error. + */ + if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + fchown(fd, uid, gid); + } } assert( fd>=0 ); if( pOutFlags ){ From af8001bfb3b38219149d1e5cecb6896fd99853bb Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Feb 2012 19:53:24 +0000 Subject: [PATCH 04/51] Another attempt to fix warnings in the randomFunc() function. FossilOrigin-Name: 768df4e11670ac704d96e3b601d009aaa2fd793a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index bf76b978d3..da4defa23a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\screating\sjournal\sfiles\s(including\s-wal\sand\s-shm\sfiles)\stry\sto\sset\sthe\nownership\sto\sbe\sthe\ssame\sas\sthe\soriginal\sdatabase.\s\sThis\swill\sprevent\sroot\nfrom\slocking\sout\sthe\soriginal\sowner\sof\sthe\sfile. -D 2012-02-11T19:23:48.068 +C Another\sattempt\sto\sfix\swarnings\sin\sthe\srandomFunc()\sfunction. +D 2012-02-11T19:53:24.226 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -137,7 +137,7 @@ F src/delete.c 51d32f0a9c880663e54ce309f52e40c325d5e112 F src/expr.c 00675123e0beec98f999aa4594d2cbe1fec33c1b F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 657212460bf5cfd3ae607d12ea62092844c227b5 -F src/func.c 482fb61e26d1c0f983e788276d3700eee26442df +F src/func.c e75f41c421f00762ab9da7dc8fb90af3972cf99f F src/global.c 4cfdca5cb0edd33c4d021baec4ede958cb2c793b F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 31142ca795005bf664f34000591e6572c72652f2 -R bf3c1343906df15f83435e1a674bd11e +P 1254dffe4071656a783cd000b1dd40c975ac18cb +R d9fa00aa327e93350ee48bb2f6ecc475 U drh -Z 10acee19c2f6805ba89a4f170956f4c5 +Z 4e53a459dae4b9322244fc7645ecba60 diff --git a/manifest.uuid b/manifest.uuid index be561debed..fb7ca58e61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1254dffe4071656a783cd000b1dd40c975ac18cb \ No newline at end of file +768df4e11670ac704d96e3b601d009aaa2fd793a \ No newline at end of file diff --git a/src/func.c b/src/func.c index 385582923d..c66ad28aba 100644 --- a/src/func.c +++ b/src/func.c @@ -416,7 +416,7 @@ static void randomFunc( ** 2s complement of that positive value. The end result can ** therefore be no less than -9223372036854775807. */ - r = -(r ^ (((sqlite3_uint64)1)<<63)); + r = -(r & LARGEST_INT64); } sqlite3_result_int64(context, r); } From 3ee3484c4ab72e70c6c29835af06e20b9063f854 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Feb 2012 21:21:17 +0000 Subject: [PATCH 05/51] Silence GCC compiler warnings about unused return value from fchown(). FossilOrigin-Name: b022547389a40930cf0d2a75f5eb293acc9fbfe0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 18 ++++++++++++------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index da4defa23a..522b3e2fc1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\sattempt\sto\sfix\swarnings\sin\sthe\srandomFunc()\sfunction. -D 2012-02-11T19:53:24.226 +C Silence\sGCC\scompiler\swarnings\sabout\sunused\sreturn\svalue\sfrom\sfchown(). +D 2012-02-11T21:21:17.335 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c f7e7b3e4f6922e3b07250a22c81da766ac2cc8fa +F src/os_unix.c 3dd0399fe024df37af2980c99f1a3c1d3cc6a782 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 1254dffe4071656a783cd000b1dd40c975ac18cb -R d9fa00aa327e93350ee48bb2f6ecc475 +P 768df4e11670ac704d96e3b601d009aaa2fd793a +R 356443deb3c82ed63a280950a7c5c72e U drh -Z 4e53a459dae4b9322244fc7645ecba60 +Z 590e6955e1a8261beb6491e7ffa39206 diff --git a/manifest.uuid b/manifest.uuid index fb7ca58e61..2b1d6181ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -768df4e11670ac704d96e3b601d009aaa2fd793a \ No newline at end of file +b022547389a40930cf0d2a75f5eb293acc9fbfe0 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 5e4ab8d248..3f3ebb6bf7 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -211,7 +211,7 @@ struct unixFile { unixInodeInfo *pInode; /* Info about locks on this inode */ int h; /* The file descriptor */ unsigned char eFileLock; /* The type of lock held on this fd */ - unsigned char ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ + unsigned short int ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ int lastErrno; /* The unix errno from last I/O error */ void *lockingContext; /* Locking style specific state */ UnixUnusedFd *pUnused; /* Pre-allocated UnixUnusedFd */ @@ -262,6 +262,7 @@ struct unixFile { #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_CHOWN 0x100 /* File ownership was changed */ /* ** Include code that is common to all os_*.c files @@ -3908,9 +3909,13 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ /* If this process is running as root, make sure that the SHM file ** is owned by the same user that owns the original database. Otherwise, ** the original owner will not be able to connect. If this process is - ** not root, the following fchown() will fail, but we don't care. + ** not root, the following fchown() will fail, but we don't care. The + ** if(){..} and the UNIXFILE_CHOWN flag are purely to silence compiler + ** warnings. */ - fchown(pShmNode->h, sStat.st_uid, sStat.st_gid); + if( fchown(pShmNode->h, sStat.st_uid, sStat.st_gid)==0 ){ + pDbFd->ctrlFlags |= UNIXFILE_CHOWN; + } /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. @@ -5119,11 +5124,12 @@ static int unixOpen( /* If this process is running as root and if creating a new rollback ** journal or WAL file, set the ownership of the journal or WAL to be ** the same as the original database. If we are not running as root, - ** then the fchown() call will fail, but that's ok - there is nothing - ** we can do about it so just ignore the error. + ** then the fchown() call will fail, but that's ok. The "if(){}" and + ** the setting of the UNIXFILE_CHOWN flag are purely to silence compiler + ** warnings from gcc. */ if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ - fchown(fd, uid, gid); + if( fchown(fd, uid, gid)==0 ){ p->ctrlFlags |= UNIXFILE_CHOWN; } } } assert( fd>=0 ); From a844f5134e4acdd2900b816c14c0fec896444ac3 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 11 Feb 2012 21:56:59 +0000 Subject: [PATCH 06/51] Revise the preprocessor directives used to handle malloc.h and _msize, in order to detect and handle the MSVC special-case automatically. FossilOrigin-Name: 90e73dc3682680847238e625d415b27fa9c59ba8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mem1.c | 17 ++++++----------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 522b3e2fc1..2203f2542b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Silence\sGCC\scompiler\swarnings\sabout\sunused\sreturn\svalue\sfrom\sfchown(). -D 2012-02-11T21:21:17.335 +C Revise\sthe\spreprocessor\sdirectives\sused\sto\shandle\smalloc.h\sand\s_msize,\sin\sorder\sto\sdetect\sand\shandle\sthe\sMSVC\sspecial-case\sautomatically. +D 2012-02-11T21:56:59.602 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -150,7 +150,7 @@ F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d F src/main.c cb099cc4864b542f97938049fc74ea8ae6845ce4 F src/malloc.c 15afac5e59b6584efe072e9933aefb4230e74f97 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c c953f3bfc8fcd31d4de2078697caefeb1dcfd7ff +F src/mem1.c b3677415e69603d6a0e7c5410a1b3731d55beda1 F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c c2c63b7067570b00bf33d751c39af24182316f7f @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 768df4e11670ac704d96e3b601d009aaa2fd793a -R 356443deb3c82ed63a280950a7c5c72e -U drh -Z 590e6955e1a8261beb6491e7ffa39206 +P b022547389a40930cf0d2a75f5eb293acc9fbfe0 +R 505ec6116aaad3f58b25fecf5b680204 +U mistachkin +Z 5a203944499c3c7e187ed1e9bd783469 diff --git a/manifest.uuid b/manifest.uuid index 2b1d6181ff..6130ce9340 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b022547389a40930cf0d2a75f5eb293acc9fbfe0 \ No newline at end of file +90e73dc3682680847238e625d415b27fa9c59ba8 \ No newline at end of file diff --git a/src/mem1.c b/src/mem1.c index 1c93a6d731..8bb8a2f1b3 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -30,12 +30,8 @@ ** can be set manually, if desired. ** If an equivalent interface exists by ** a different name, using a separate -D -** option to rename it. This symbol will -** be enabled automatically on windows -** systems, and malloc_usable_size() will -** be redefined to _msize(), unless the -** SQLITE_WITHOUT_MSIZE macro is defined. -** +** option to rename it. +** ** SQLITE_WITHOUT_ZONEMALLOC Some older macs lack support for the zone ** memory allocator. Set this symbol to enable ** building on older macs. @@ -55,13 +51,11 @@ #ifdef SQLITE_SYSTEM_MALLOC /* -** Windows systems have malloc_usable_size() but it is called _msize(). +** The MSVCRT has malloc_usable_size() but it is called _msize(). ** The use of _msize() is automatic, but can be disabled by compiling ** with -DSQLITE_WITHOUT_MSIZE */ -#if !defined(HAVE_MALLOC_USABLE_SIZE) && SQLITE_OS_WIN \ - && !defined(SQLITE_WITHOUT_MSIZE) -# define HAVE_MALLOC_USABLE_SIZE 1 +#if defined(_MSC_VER) && !defined(SQLITE_WITHOUT_MSIZE) # define SQLITE_MALLOCSIZE _msize #endif @@ -91,7 +85,8 @@ static malloc_zone_t* _sqliteZone_; #define SQLITE_FREE(x) free(x) #define SQLITE_REALLOC(x,y) realloc((x),(y)) -#if defined(HAVE_MALLOC_H) && defined(HAVE_MALLOC_USABLE_SIZE) +#if (defined(_MSC_VER) && !defined(SQLITE_WITHOUT_MSIZE)) \ + || (defined(HAVE_MALLOC_H) && defined(HAVE_MALLOC_USABLE_SIZE)) # include /* Needed for malloc_usable_size on linux */ #endif #ifdef HAVE_MALLOC_USABLE_SIZE From b482a4462c134bb068b33071ff236c012ae0ea64 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 11 Feb 2012 22:19:26 +0000 Subject: [PATCH 07/51] Cleanup some directories and files left by the MinGW build, even when using the MSVC Makefile. FossilOrigin-Name: 3ab89e255d1f2f8004d3256515ac43f74e57f39b --- Makefile.msc | 3 +++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index f40936d915..e95b443fe1 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -979,9 +979,12 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c clean: del /Q *.lo *.ilk *.lib *.obj *.pdb sqlite3.exe libsqlite3.lib + del /Q *.da *.bb *.bbg gmon.out del /Q sqlite3.h opcodes.c opcodes.h del /Q lemon.exe lempar.c parse.* del /Q mkkeywordhash.exe keywordhash.h + -rmdir /Q/S .deps + -rmdir /Q/S .libs -rmdir /Q/S tsrc del /Q .target_source del /Q tclsqlite3.exe diff --git a/manifest b/manifest index 2203f2542b..913cda3a67 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Revise\sthe\spreprocessor\sdirectives\sused\sto\shandle\smalloc.h\sand\s_msize,\sin\sorder\sto\sdetect\sand\shandle\sthe\sMSVC\sspecial-case\sautomatically. -D 2012-02-11T21:56:59.602 +C Cleanup\ssome\sdirectories\sand\sfiles\sleft\sby\sthe\sMinGW\sbuild,\seven\swhen\susing\sthe\sMSVC\sMakefile. +D 2012-02-11T22:19:26.264 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc dcad80fa69f17d46fe6778ba873fc108ca16298d +F Makefile.msc 3a5582a858b8071af43cd459bd757f7d0748f66a F Makefile.vxworks 1deb39c8bb047296c30161ffa10c1b5423e632f9 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION bb4c2a86abe53ea3c1d6ea515b69a426040e2414 @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P b022547389a40930cf0d2a75f5eb293acc9fbfe0 -R 505ec6116aaad3f58b25fecf5b680204 +P 90e73dc3682680847238e625d415b27fa9c59ba8 +R 6d21c0cf3bea5fa0f98de70e56cbbd21 U mistachkin -Z 5a203944499c3c7e187ed1e9bd783469 +Z df222febfd73cd10d8b058c8ec5473c1 diff --git a/manifest.uuid b/manifest.uuid index 6130ce9340..37aedc15f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90e73dc3682680847238e625d415b27fa9c59ba8 \ No newline at end of file +3ab89e255d1f2f8004d3256515ac43f74e57f39b \ No newline at end of file From 23c4b9732940cbd22b9203514b655e5e992f0900 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Feb 2012 23:55:15 +0000 Subject: [PATCH 08/51] Make fchown() an overrideable system call in os_unix.c. FossilOrigin-Name: 98efac9630ca96ba0497c8e206dc5022612ff381 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 9 ++++++--- test/syscall.test | 2 +- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 913cda3a67..c0a205521f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cleanup\ssome\sdirectories\sand\sfiles\sleft\sby\sthe\sMinGW\sbuild,\seven\swhen\susing\sthe\sMSVC\sMakefile. -D 2012-02-11T22:19:26.264 +C Make\sfchown()\san\soverrideable\ssystem\scall\sin\sos_unix.c. +D 2012-02-11T23:55:15.455 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 3dd0399fe024df37af2980c99f1a3c1d3cc6a782 +F src/os_unix.c 55fbf23bb9807412a6ec95c5cc489222e56a4d58 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a @@ -709,7 +709,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test 265cda616f56a297406728ee1e74c9b4a93aa6dd +F test/syscall.test 8a1bd9575ea1e8bdc0513fc5be7753fa4c3c04fb F test/sysfault.test c79441d88d23696fbec7b147dba98d42a04f523f F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 90e73dc3682680847238e625d415b27fa9c59ba8 -R 6d21c0cf3bea5fa0f98de70e56cbbd21 -U mistachkin -Z df222febfd73cd10d8b058c8ec5473c1 +P 3ab89e255d1f2f8004d3256515ac43f74e57f39b +R c7e080a7342ae25512a1d89fb1e6a36a +U drh +Z 57dd073c92c9b4505dfd0c4c6af07210 diff --git a/manifest.uuid b/manifest.uuid index 37aedc15f1..f192e997a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ab89e255d1f2f8004d3256515ac43f74e57f39b \ No newline at end of file +98efac9630ca96ba0497c8e206dc5022612ff381 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 3f3ebb6bf7..c6f71adc8a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -419,6 +419,9 @@ static struct unix_syscall { { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) + { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, +#define osFchown ((int(*)(const char*,uid_t,gid_t))aSyscall[20].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -3913,7 +3916,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** if(){..} and the UNIXFILE_CHOWN flag are purely to silence compiler ** warnings. */ - if( fchown(pShmNode->h, sStat.st_uid, sStat.st_gid)==0 ){ + if( osFchown(pShmNode->h, sStat.st_uid, sStat.st_gid)==0 ){ pDbFd->ctrlFlags |= UNIXFILE_CHOWN; } @@ -5129,7 +5132,7 @@ static int unixOpen( ** warnings from gcc. */ if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ - if( fchown(fd, uid, gid)==0 ){ p->ctrlFlags |= UNIXFILE_CHOWN; } + if( osFchown(fd, uid, gid)==0 ){ p->ctrlFlags |= UNIXFILE_CHOWN; } } } assert( fd>=0 ); @@ -6839,7 +6842,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==20 ); + assert( ArraySize(aSyscall)==21 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/test/syscall.test b/test/syscall.test index dde4b467a2..da7ecd53ba 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -60,7 +60,7 @@ foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir - statvfs + statvfs fchown } { if {[test_syscall exists $s]} {lappend syscall_list $s} } From d3eaebd4146feb550304443430f4f57ce8bb5245 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Feb 2012 08:50:23 +0000 Subject: [PATCH 09/51] Fix a typo in the type of the fchown() function in os_unix.c. FossilOrigin-Name: bfbfe05b81919ecc3d6e7be4c24994f795f16582 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c0a205521f..7059003de2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sfchown()\san\soverrideable\ssystem\scall\sin\sos_unix.c. -D 2012-02-11T23:55:15.455 +C Fix\sa\stypo\sin\sthe\stype\sof\sthe\sfchown()\sfunction\sin\sos_unix.c. +D 2012-02-13T08:50:23.792 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 55fbf23bb9807412a6ec95c5cc489222e56a4d58 +F src/os_unix.c fd5875e602286c8ecd96a0049bd7869c30721df0 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 3ab89e255d1f2f8004d3256515ac43f74e57f39b -R c7e080a7342ae25512a1d89fb1e6a36a -U drh -Z 57dd073c92c9b4505dfd0c4c6af07210 +P 98efac9630ca96ba0497c8e206dc5022612ff381 +R c28baf50a89887974b43cddbc49224fb +U dan +Z fe60136d21d83b01836f509b8ebeab38 diff --git a/manifest.uuid b/manifest.uuid index f192e997a1..b37b52966d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98efac9630ca96ba0497c8e206dc5022612ff381 \ No newline at end of file +bfbfe05b81919ecc3d6e7be4c24994f795f16582 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index c6f71adc8a..31fb1b40d9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -420,7 +420,7 @@ static struct unix_syscall { #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, -#define osFchown ((int(*)(const char*,uid_t,gid_t))aSyscall[20].pCurrent) +#define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) }; /* End of the overrideable system calls */ From 2f56da3f262751a306242d61add9712cd3833437 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Feb 2012 10:00:35 +0000 Subject: [PATCH 10/51] Changes to various test scripts so that veryquick.test runs with OMIT_COMPOUND_SELECT defined. FossilOrigin-Name: 76bb649ee2633226324130f5898622c348f93769 --- manifest | 76 +++++++++++++++++++-------------------- manifest.uuid | 2 +- test/cache.test | 2 +- test/dbstatus.test | 5 +++ test/distinct.test | 5 +++ test/e_delete.test | 5 +++ test/e_expr.test | 4 +++ test/e_fkey.test | 4 +-- test/e_insert.test | 5 +++ test/e_select.test | 5 +++ test/e_update.test | 8 ++--- test/e_vacuum.test | 2 +- test/eqp.test | 5 +++ test/in.test | 2 ++ test/insert.test | 26 +++++++------- test/join6.test | 42 +++++++++++----------- test/minmax4.test | 5 +++ test/randexpr1.test | 5 +++ test/savepoint.test | 8 ++--- test/selectC.test | 2 +- test/stat.test | 2 +- test/tkt-02a8e81d44.test | 5 +++ test/tkt-38cb5df375.test | 5 +++ test/tkt-3a77c9714e.test | 5 +++ test/tkt-7bbfb7d442.test | 10 +++--- test/tkt-80ba201079.test | 24 +++++++------ test/tkt-b72787b1.test | 5 +++ test/tkt-d82e3f3721.test | 5 +++ test/tkt-f777251dc7a.test | 5 +++ test/tkt3527.test | 5 +++ test/tkt3773.test | 5 +++ test/trigger1.test | 2 +- test/vtabD.test | 10 +++--- test/vtab_shared.test | 36 ++++++++++--------- test/where9.test | 2 +- 35 files changed, 218 insertions(+), 126 deletions(-) diff --git a/manifest b/manifest index 7059003de2..bdb31adf3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\stype\sof\sthe\sfchown()\sfunction\sin\sos_unix.c. -D 2012-02-13T08:50:23.792 +C Changes\sto\svarious\stest\sscripts\sso\sthat\sveryquick.test\sruns\swith\sOMIT_COMPOUND_SELECT\sdefined. +D 2012-02-13T10:00:35.138 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -312,7 +312,7 @@ F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 -F test/cache.test 754baab2f18089fc9bcba7afaeb4dc907c6c6de2 +F test/cache.test f64136b0893c293d0b910ed057b3b711249099a7 F test/capi2.test 835d4cee9f542ea50fa8d01f3fe6de80b0627360 F test/capi3.test 7200dff6acb17b9a4b6f9918f554eaae04968ddd F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 @@ -364,7 +364,7 @@ F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test a18a2ce81add84b17b06559e82ad7bb91bc6ddff -F test/dbstatus.test 3e978f8bdb2362a36a4be63c36a59f542c4cc2a3 +F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b F test/dbstatus2.test dc57b0d9610851c0ff58a8e1b5b191678398b72a F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 @@ -374,28 +374,28 @@ F test/descidx1.test 533dcbda614b0463b0ea029527fd27e5a9ab2d66 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test 76908ed038c5186ffb8acf5954ed64e22056f311 +F test/distinct.test 8c6d12ba53ee8351a5b2d47628acdfad1fc97743 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test 48598b15e8fe6554d301e7b65a10c9851f177e84 -F test/e_delete.test ec168cd4b08d681e6d5847f462203755ad647532 +F test/e_delete.test 89aa84d3d1bd284a0689ede04bce10226a5aeaa5 F test/e_droptrigger.test afd5c4d27dec607f5997a66bf7e2498a082cb235 F test/e_dropview.test 583411e470458c5d76148542cfb5a5fa84c8f93e -F test/e_expr.test 4e4399006b3d1ab333721b8e386cabb9fb6d5a89 -F test/e_fkey.test 38039b840ab19331000b0f0eb1d82baa7208a67a +F test/e_expr.test 5489424d3d9a452ac3701cdf4b680ae31a157894 +F test/e_fkey.test 057eed81a41a2b21b1790032f4e8aaba0b2b0e17 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 -F test/e_insert.test 234242b71855af8e8a9b1e141c3533f6d43d8645 +F test/e_insert.test 92d2dab07366aef112f53af4539e30559f5d35a7 F test/e_reindex.test dfedfc32c5a282b0596c6537cbcd4217fbb1a216 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 -F test/e_select.test 99202f99a9a3273c6fb0d2e7592b98faeb6c206e +F test/e_select.test f5d4b81205701deacfae42051ae200969c41d2c0 F test/e_select2.test 5c3d3da19c7b3e90ae444579db2b70098599ab92 -F test/e_update.test dba988a4d079156549a40854074ba4890b0a4577 +F test/e_update.test 161d5dc6a3ed9dd08f5264d13e20735d7a89f00c F test/e_uri.test 6f35b491f80dac005c8144f38b2dfb4d96483596 -F test/e_vacuum.test 5296e25ef871965bac010b9da083dd7e4734526e +F test/e_vacuum.test 331da289ae186656cf5f2eb27f577a89c0c221af F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 796c59832e2b9a52842f382ffda8f3e989db03ad F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test f14fadd76da53405e9885e2431cacf7191d83cdb +F test/eqp.test 6a389bba6ea113fd5179515001be788a38d53ec7 F test/errmsg.test 3bb606db9d040cc6854459f8f5e5a2bcd9b7fd2a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb @@ -507,7 +507,7 @@ F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 F test/fuzzer1.test ddfb04f3bd5cfdda3b1aa15b78d3ad055c9cc50f F test/hook.test 5f3749de6462a6b87b4209b74adf7df5ac2df639 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 -F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d +F test/in.test a7b8a0f43da81cd08645b7a710099ffe9ad1126b F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 @@ -526,7 +526,7 @@ F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026 F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 -F test/insert.test d540650825c98d8082d32f786c611d70e1c21a80 +F test/insert.test 489aa12a027c83d291f5034a83c8c32e6be1dca2 F test/insert2.test 4f3a04d168c728ed5ec2c88842e772606c7ce435 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 F test/insert4.test 87f6798f31d60c4e177622fcc3663367e6ecbd90 @@ -545,7 +545,7 @@ F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 86675fc2919269aa923c84dd00ee4249b97990fe -F test/join6.test bf82cf3f979e9eade83ad0d056a66c5ed71d1901 +F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 8b71ef1ed5798bdc0e6eb616d8694e2c2c188d4d F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 F test/journal3.test 6fd28532c88b447db844186bc190523108b6dbb4 @@ -601,7 +601,7 @@ F test/memsubsys2.test 3a1c1a9de48e5726faa85108b02459fae8cb9ee9 F test/minmax.test 722d80816f7e096bf2c04f4111f1a6c1ba65453d F test/minmax2.test 33504c01a03bd99226144e4b03f7631a274d66e0 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 -F test/minmax4.test c1fa9505fd135007fdb1fb699334fb3d4ea7952e +F test/minmax4.test 536a3360470633a177e42fbc19660d146b51daef F test/misc1.test 55cb2bfbf4a8cd61f4be1effc30426ad41696bff F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test fe55130a43e444ee75e2156ff75dc96e964b5738 @@ -645,7 +645,7 @@ F test/quota.test af47d25c166aa7b33ef25f21bb7f2afb29d82c77 F test/quota2.test 1b8df088e604f2df573f96e726b5e518cb0cddaa F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 -F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a +F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test c267d050a1d9a6a321de502b737daf28821a518d F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a @@ -654,7 +654,7 @@ F test/rollback.test a1b4784b864331eae8b2a98c189efa2a8b11ff07 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test e58e0acef38b527ed1b0b70d3ada588f804af287 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 -F test/savepoint.test e575217b07d6a6e895e66f4eda076570815e0027 +F test/savepoint.test 42ba63b632df4ce1bef8acdba62e4182d6505125 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 @@ -676,7 +676,7 @@ F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test 74c0fb2c6eecb0219cbed0cbe3df136f8fbf9343 F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 -F test/selectC.test f9bf1bc4581b5b8158caa6e4e4f682acb379fb25 +F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 F test/shared.test 34945a516532b11182c3eb26e31247eee3c9ae48 F test/shared2.test 8f71d4eb4d5261280de92284df74172545c852cc @@ -701,7 +701,7 @@ F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 -F test/stat.test 36bc951bdc2beac4224cc54396fd6a7dc65336f4 +F test/stat.test 08e8185b3fd5b010c90d7ad82b9dd4ea1cbf14b0 F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 F test/subquery.test b524f57c9574b2c0347045b4510ef795d4686796 F test/subquery2.test edcad5c118f0531c2e21bf16a09bbb105252d4cd @@ -729,15 +729,15 @@ F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 F test/threadtest3.c 0ed13e09690f6204d7455fac3b0e8ece490f6eef -F test/tkt-02a8e81d44.test 58494de77be2cf249228ada3f313fa399821c6ab +F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2d1a5c67d.test b028a811049eb472cb2d3a43fc8ce4f6894eebda F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 1f714c14b6682c5db715e0bda347926a3456f7a9 F test/tkt-313723c356.test c47f8a9330523e6f35698bf4489bcb29609b53ac -F test/tkt-38cb5df375.test 9e9b19857dba0896a8efdaf334d405ba423492f2 +F test/tkt-38cb5df375.test f3cc8671f1eb604d4ae9cf886ed4366bec656678 F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7 -F test/tkt-3a77c9714e.test 1675c22a5be71d7fa026e5db5daeeb4dd64f7824 +F test/tkt-3a77c9714e.test 32bb28afa8c63fc76e972e996193139b63551ed9 F test/tkt-3fe897352e.test 10de1a67bd5c66b238a4c96abe55531b37bb4f00 F test/tkt-4a03edc4c8.test 2865e4edbc075b954daa82f8da7cc973033ec76e F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 @@ -746,8 +746,8 @@ F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f F test/tkt-752e1646fc.test ea78d88d14fe9866bdd991c634483334639e13bf F test/tkt-78e04e52ea.test ab52f0c1e2de6e46c910f4cc16b086bba05952b7 -F test/tkt-7bbfb7d442.test 8e7658f77d1ccea9d88dc9e255d3ed7fb68f8bdf -F test/tkt-80ba201079.test a09684db1a0bd55b8838f606adccee456a51ddbf +F test/tkt-7bbfb7d442.test dfa5c8097a8c353ae40705d6cddeb1f99c18b81a +F test/tkt-80ba201079.test 9eb040d81c404f56838a6af93593f42790def63f F test/tkt-80e031a00f.test 9a154173461a4dbe2de49cda73963e04842d52f7 F test/tkt-8454a207b9.test c583a9f814a82a2b5ba95207f55001c9f0cd816c F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 @@ -755,15 +755,15 @@ F test/tkt-94c04eaadb.test be5ea61cb04dfdc047d19b5c5a9e75fa3da67a7f F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 F test/tkt-b1d3a2e531.test 610ef582413171b379652663111b1f996d9f8f78 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 -F test/tkt-b72787b1.test e6b62b2b2785c04d0d698d6a603507e384165049 +F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-c48d99d690.test bed446e3513ae10eec1b86fdd186ef750226c408 F test/tkt-cbd054fa6b.test bd9fb546f63bc0c79d1776978d059fa51c5b1c63 F test/tkt-d11f09d36e.test fb44f7961aa6d4b632fb7b9768239832210b5fc7 F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 -F test/tkt-d82e3f3721.test 731359dfdcdb36fea0559cd33fec39dd0ceae8e6 +F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test 669bb076f2ac573c7398ce00f40cd0ca502043a9 -F test/tkt-f777251dc7a.test 6f24c053bc5cdb7e1e19be9a72c8887cf41d5e87 +F test/tkt-f777251dc7a.test a3121f92f49af447572f7a3cde9f07806443c75d F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 1da0ed15ec2c7749fb5ce2828cd69d07153ad9f4 F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 @@ -822,7 +822,7 @@ F test/tkt3461.test 228ea328a5a21e8663f80ee3d212a6ad92549a19 F test/tkt3493.test 1686cbde85f8721fc1bdc0ee72f2ef2f63139218 F test/tkt3508.test d75704db9501625c7f7deec119fcaf1696aefb7d F test/tkt3522.test 22ce2ebbcb04a6be56c0977d405c207967318fd6 -F test/tkt3527.test ee4af96183579565987e58873a7490bc04934ffb +F test/tkt3527.test 9e8f28a706c772d5a7cd1020c946fab6c74e3ae0 F test/tkt3541.test 5dc257bde9bc833ab9cc6844bf170b998dbb950a F test/tkt3554.test f599967f279077bace39220cbe76085c7b423725 F test/tkt3581.test 1966b7193f1e3f14951cce8c66907ae69454e9a3 @@ -833,7 +833,7 @@ F test/tkt3731.test 0c5f4cbffe102d43c3b2188af91a9e36348f974b F test/tkt3757.test 10cd679a88675c880533083fc79ac04324525595 F test/tkt3761.test b95ea9c98f21cf91325f18a984887e62caceab33 F test/tkt3762.test 2a9f3b03df44ec49ec0cfa8d5da6574c2a7853df -F test/tkt3773.test 430b06567ce40285dfd2c4834a2a61816403efeb +F test/tkt3773.test 7bca904d2a647a6a4a291bd86d7fd7c73855b789 F test/tkt3791.test a6624b9a80b216a26cf473607f42f3e51898c267 F test/tkt3793.test d90ffd75c52413908d15e1c44fc2ea9c80fcc449 F test/tkt3810.test 90fa0635dfa7da9680c8cd3513350a49b3a8ae12 @@ -857,7 +857,7 @@ F test/trace2.test 962175290996d5f06dc4402ca218bbfc7df4cb20 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test d728abaa318ca364dc370e06576aa7e5fbed7e97 -F test/trigger1.test 38524d80ac26c232d23ecec4b037eb60fb67eedd +F test/trigger1.test de42feb7cd442787d38185ae74f5a1d7afa400cb F test/trigger2.test 834187beafd1db383af0c659cfa49b0576832816 F test/trigger3.test d2c60d8be271c355d61727411e753181e877230a F test/trigger4.test 74700b76ebf3947b2f7a92405141eb2cf2a5d359 @@ -899,12 +899,12 @@ F test/vtab9.test ea58d2b95d61955f87226381716b2d0b1d4e4f9b F test/vtabA.test c86e1990b7e1e2bb34602a06fffa4c69f2b516dc F test/vtabB.test 04df5dc531b9f44d9ca65b9c1b79f12b5922a796 F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 -F test/vtabD.test 74167b1578e5886fe4c886d6bef2fd1406444c42 +F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 -F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d +F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 F test/wal.test edefe316b4125d7f68004ea53c5e73c398d436cc F test/wal2.test f11883dd3cb7f647c5d2acfd7b5c6d4ba5770cc9 F test/wal3.test 6504bbf348b2d6dfade64a064f1050fd617e8706 @@ -937,7 +937,7 @@ F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 814d7373413398e074f134cff5f8872e2c08bd3b F test/where8.test a6c740fd286d7883e274e17b6230a9d672a7ab1f F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test bed66dcfc69a54a99661c0c9906189cb5e58f4e2 +F test/where9.test cd4ee5e455799ddba7041e5ac539044bb24e3874 F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test 13ff5ec0dba407c0e0c075980c75b3275a6774e5 @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 98efac9630ca96ba0497c8e206dc5022612ff381 -R c28baf50a89887974b43cddbc49224fb +P bfbfe05b81919ecc3d6e7be4c24994f795f16582 +R 6250e4472fb78736176921352e48e2f9 U dan -Z fe60136d21d83b01836f509b8ebeab38 +Z f0319cf74449ae83da7435ab8330767c diff --git a/manifest.uuid b/manifest.uuid index b37b52966d..896ec58a38 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bfbfe05b81919ecc3d6e7be4c24994f795f16582 \ No newline at end of file +76bb649ee2633226324130f5898622c348f93769 \ No newline at end of file diff --git a/test/cache.test b/test/cache.test index 8d801f0599..f81948ba2c 100644 --- a/test/cache.test +++ b/test/cache.test @@ -14,7 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable {!pager_pragmas} { +ifcapable !pager_pragmas||!compound { finish_test return } diff --git a/test/dbstatus.test b/test/dbstatus.test index 5fc8be0426..9793df3f4e 100644 --- a/test/dbstatus.test +++ b/test/dbstatus.test @@ -15,6 +15,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + # Memory statistics must be enabled for this test. db close sqlite3_shutdown diff --git a/test/distinct.test b/test/distinct.test index 3fb9c8478a..3bc8337942 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -15,6 +15,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + set testprefix distinct diff --git a/test/e_delete.test b/test/e_delete.test index 91459810f9..31bb3249ba 100644 --- a/test/e_delete.test +++ b/test/e_delete.test @@ -15,6 +15,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + proc do_delete_tests {args} { uplevel do_select_tests $args } diff --git a/test/e_expr.test b/test/e_expr.test index c770f1c410..74d0c40712 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -17,6 +17,10 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl +ifcapable !compound { + finish_test + return +} proc do_expr_test {tn expr type value} { uplevel do_execsql_test $tn [list "SELECT typeof($expr), $expr"] [ diff --git a/test/e_fkey.test b/test/e_fkey.test index ae789d5072..5b27e03b5a 100644 --- a/test/e_fkey.test +++ b/test/e_fkey.test @@ -2325,7 +2325,7 @@ do_test e_fkey-51.1 { do_test e_fkey-51.2 { execsql { UPDATE parent SET x = 22; - SELECT * FROM parent UNION ALL SELECT 'xxx' UNION ALL SELECT a FROM child; + SELECT * FROM parent ; SELECT 'xxx' ; SELECT a FROM child; } } {22 21 23 xxx 22} do_test e_fkey-51.3 { @@ -2335,7 +2335,7 @@ do_test e_fkey-51.3 { INSERT INTO parent VALUES(-1); INSERT INTO child VALUES(-1); UPDATE parent SET x = 22; - SELECT * FROM parent UNION ALL SELECT 'xxx' UNION ALL SELECT a FROM child; + SELECT * FROM parent ; SELECT 'xxx' ; SELECT a FROM child; } } {22 23 21 xxx 23} diff --git a/test/e_insert.test b/test/e_insert.test index e96b1a6f6e..a10d5283de 100644 --- a/test/e_insert.test +++ b/test/e_insert.test @@ -18,6 +18,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + # Organization of tests: # # e_insert-0.*: Test the syntax diagram. diff --git a/test/e_select.test b/test/e_select.test index 103472cd92..e5949af67b 100644 --- a/test/e_select.test +++ b/test/e_select.test @@ -16,6 +16,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_execsql_test e_select-1.0 { CREATE TABLE t1(a, b); INSERT INTO t1 VALUES('a', 'one'); diff --git a/test/e_update.test b/test/e_update.test index 6c0e5f8aa6..230c97fdfa 100644 --- a/test/e_update.test +++ b/test/e_update.test @@ -381,11 +381,9 @@ do_execsql_test e_update-2.2.X { # attached). # do_execsql_test e_update-2.3.0 { - SELECT 'main', tbl_name FROM main.sqlite_master WHERE type = 'table' - UNION ALL - SELECT 'temp', tbl_name FROM sqlite_temp_master WHERE type = 'table' - UNION ALL - SELECT 'aux', tbl_name FROM aux.sqlite_master WHERE type = 'table' + SELECT 'main', tbl_name FROM main.sqlite_master WHERE type = 'table'; + SELECT 'temp', tbl_name FROM sqlite_temp_master WHERE type = 'table'; + SELECT 'aux', tbl_name FROM aux.sqlite_master WHERE type = 'table'; } [list {*}{ main t1 main t2 diff --git a/test/e_vacuum.test b/test/e_vacuum.test index aa7870fd10..bad12d3c4d 100644 --- a/test/e_vacuum.test +++ b/test/e_vacuum.test @@ -122,7 +122,7 @@ foreach {tn avmode sz} { # e_vacuum-1.2.4 - Verify that t1 and its indexes are now much # less fragmented. # -ifcapable vtab { +ifcapable vtab&&compound { create_db register_dbstat_vtab db do_execsql_test e_vacuum-1.2.1 { diff --git a/test/eqp.test b/test/eqp.test index 91a18d0026..0e663f0a38 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -13,6 +13,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + set testprefix eqp #------------------------------------------------------------------------- diff --git a/test/in.test b/test/in.test index 2c38a0f4ca..108f35589c 100644 --- a/test/in.test +++ b/test/in.test @@ -431,6 +431,7 @@ do_test in-12.9 { } {1 {SELECTs to the left and right of INTERSECT do not have the same number of result columns}} } +ifcapable compound { do_test in-12.10 { catchsql { SELECT * FROM t2 WHERE a IN ( @@ -459,6 +460,7 @@ do_test in-12.13 { ); } } {1 {only a single result allowed for a SELECT that is part of an expression}} +}; #ifcapable compound #------------------------------------------------------------------------ diff --git a/test/insert.test b/test/insert.test index 05718d556f..e00b9a8028 100644 --- a/test/insert.test +++ b/test/insert.test @@ -388,18 +388,20 @@ do_test insert-9.2 { # Multiple VALUES clauses # -do_test insert-10.1 { - execsql { - CREATE TABLE t10(a,b,c); - INSERT INTO t10 VALUES(1,2,3), (4,5,6), (7,8,9); - SELECT * FROM t10; - } -} {1 2 3 4 5 6 7 8 9} -do_test insert-10.2 { - catchsql { - INSERT INTO t10 VALUES(11,12,13), (14,15); - } -} {1 {all VALUES must have the same number of terms}} +ifcapable compound { + do_test insert-10.1 { + execsql { + CREATE TABLE t10(a,b,c); + INSERT INTO t10 VALUES(1,2,3), (4,5,6), (7,8,9); + SELECT * FROM t10; + } + } {1 2 3 4 5 6 7 8 9} + do_test insert-10.2 { + catchsql { + INSERT INTO t10 VALUES(11,12,13), (14,15); + } + } {1 {all VALUES must have the same number of terms}} +} integrity_check insert-99.0 diff --git a/test/join6.test b/test/join6.test index 4f65dcb446..7fbf508e57 100644 --- a/test/join6.test +++ b/test/join6.test @@ -124,26 +124,28 @@ do_test join6-3.6 { } } {1 91 92 3 93 5 91 2 93 94 4 95 6 99} -do_test join6-4.1 { - execsql { - SELECT * FROM - (SELECT 1 AS a, 91 AS x, 92 AS y UNION SELECT 2, 93, 94) - NATURAL JOIN t2 NATURAL JOIN t3 - } -} {1 91 92 3 93 5} -do_test join6-4.2 { - execsql { - SELECT * FROM t1 NATURAL JOIN - (SELECT 3 AS b, 92 AS y, 93 AS z UNION SELECT 4, 94, 95) - NATURAL JOIN t3 - } -} {1 91 92 3 93 5} -do_test join6-4.3 { - execsql { - SELECT * FROM t1 NATURAL JOIN t2 NATURAL JOIN - (SELECT 5 AS c, 91 AS x, 93 AS z UNION SELECT 6, 99, 95) - } -} {1 91 92 3 93 5} +ifcapable compound { + do_test join6-4.1 { + execsql { + SELECT * FROM + (SELECT 1 AS a, 91 AS x, 92 AS y UNION SELECT 2, 93, 94) + NATURAL JOIN t2 NATURAL JOIN t3 + } + } {1 91 92 3 93 5} + do_test join6-4.2 { + execsql { + SELECT * FROM t1 NATURAL JOIN + (SELECT 3 AS b, 92 AS y, 93 AS z UNION SELECT 4, 94, 95) + NATURAL JOIN t3 + } + } {1 91 92 3 93 5} + do_test join6-4.3 { + execsql { + SELECT * FROM t1 NATURAL JOIN t2 NATURAL JOIN + (SELECT 5 AS c, 91 AS x, 93 AS z UNION SELECT 6, 99, 95) + } + } {1 91 92 3 93 5} +} diff --git a/test/minmax4.test b/test/minmax4.test index 12f56a86c3..0d8305b5ff 100644 --- a/test/minmax4.test +++ b/test/minmax4.test @@ -20,6 +20,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test minmax4-1.1 { db eval { CREATE TABLE t1(p,q); diff --git a/test/randexpr1.test b/test/randexpr1.test index 10a1d17c39..7a98f0be0f 100644 --- a/test/randexpr1.test +++ b/test/randexpr1.test @@ -22,6 +22,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + # Create test data # do_test randexpr1-1.1 { diff --git a/test/savepoint.test b/test/savepoint.test index 3bbbaaae31..6163e2e8a9 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -649,10 +649,8 @@ if {[wal_is_wal_mode]==0} { CREATE TABLE main.t1(x, y); CREATE TABLE aux1.t2(x, y); CREATE TABLE aux2.t3(x, y); - SELECT name FROM sqlite_master - UNION ALL - SELECT name FROM aux1.sqlite_master - UNION ALL + SELECT name FROM sqlite_master; + SELECT name FROM aux1.sqlite_master; SELECT name FROM aux2.sqlite_master; } } {t1 t2 t3} @@ -691,7 +689,7 @@ if {[wal_is_wal_mode]==0} { execsql { PRAGMA lock_status } } [list main reserved temp $templockstate aux1 reserved aux2 reserved] do_test savepoint-10.2.9 { - execsql { SELECT 'a', * FROM t1 UNION ALL SELECT 'b', * FROM t3 } + execsql { SELECT 'a', * FROM t1 ; SELECT 'b', * FROM t3 } } {a 1 2 b 3 4} do_test savepoint-10.2.9 { execsql { diff --git a/test/selectC.test b/test/selectC.test index fd384053ce..dedac41fc4 100644 --- a/test/selectC.test +++ b/test/selectC.test @@ -151,7 +151,7 @@ do_test selectC-1.14.2 { # The following query used to leak memory. Verify that has been fixed. # -ifcapable trigger { +ifcapable trigger&&compound { do_test selectC-2.1 { catchsql { CREATE TABLE t21a(a,b); diff --git a/test/stat.test b/test/stat.test index 21726eb258..926d9b7406 100644 --- a/test/stat.test +++ b/test/stat.test @@ -15,7 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !vtab { +ifcapable !vtab||!compound { finish_test return } diff --git a/test/tkt-02a8e81d44.test b/test/tkt-02a8e81d44.test index 4a48fb029e..7ca9866414 100644 --- a/test/tkt-02a8e81d44.test +++ b/test/tkt-02a8e81d44.test @@ -17,6 +17,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt-02a838-1.1 { execsql { CREATE TABLE t1(a); diff --git a/test/tkt-38cb5df375.test b/test/tkt-38cb5df375.test index 47b0b55106..e5e02671b7 100644 --- a/test/tkt-38cb5df375.test +++ b/test/tkt-38cb5df375.test @@ -16,6 +16,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt-38cb5df375.0 { execsql { CREATE TABLE t1(a); diff --git a/test/tkt-3a77c9714e.test b/test/tkt-3a77c9714e.test index d77ee42bb3..6eaec16cdd 100644 --- a/test/tkt-3a77c9714e.test +++ b/test/tkt-3a77c9714e.test @@ -16,6 +16,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + set testprefix "tkt-3a77c9714e" do_execsql_test 1.1 { diff --git a/test/tkt-7bbfb7d442.test b/test/tkt-7bbfb7d442.test index e560a0d4d5..dcb9b16058 100644 --- a/test/tkt-7bbfb7d442.test +++ b/test/tkt-7bbfb7d442.test @@ -49,10 +49,12 @@ do_execsql_test 1.2 { do_execsql_test 1.3 { DELETE FROM t3 } -do_execsql_test 1.4 { - INSERT INTO t3(t3_a) SELECT 1 UNION SELECT 2 UNION SELECT 3; - SELECT * FROM t3; -} {1 I 2 II 3 III} +ifcapable compound { + do_execsql_test 1.4 { + INSERT INTO t3(t3_a) SELECT 1 UNION SELECT 2 UNION SELECT 3; + SELECT * FROM t3; + } {1 I 2 II 3 III} +} diff --git a/test/tkt-80ba201079.test b/test/tkt-80ba201079.test index 95e99b5a5b..0122e95f2f 100644 --- a/test/tkt-80ba201079.test +++ b/test/tkt-80ba201079.test @@ -164,11 +164,13 @@ do_execsql_test 303 { (b='B' AND c IN (SELECT c FROM t1)) } {A B C D E} -do_execsql_test 304 { - SELECT * FROM t1, t2 WHERE - (a='A' AND d='E') OR - (b='B' AND c IN (SELECT 'B' UNION SELECT 'C' UNION SELECT 'D')) -} {A B C D E} +ifcapable compound { + do_execsql_test 304 { + SELECT * FROM t1, t2 WHERE + (a='A' AND d='E') OR + (b='B' AND c IN (SELECT 'B' UNION SELECT 'C' UNION SELECT 'D')) + } {A B C D E} +} do_execsql_test 305 { SELECT * FROM t1, t2 WHERE @@ -182,10 +184,12 @@ do_execsql_test 306 { (a='A' AND d='E') } {A B C D E} -do_execsql_test 307 { - SELECT * FROM t1, t2 WHERE - (b='B' AND c IN (SELECT 'B' UNION SELECT 'C' UNION SELECT 'D')) OR - (a='A' AND d='E') -} {A B C D E} +ifcapable compound { + do_execsql_test 307 { + SELECT * FROM t1, t2 WHERE + (b='B' AND c IN (SELECT 'B' UNION SELECT 'C' UNION SELECT 'D')) OR + (a='A' AND d='E') + } {A B C D E} +} finish_test diff --git a/test/tkt-b72787b1.test b/test/tkt-b72787b1.test index 11ea41ed7a..dea3f49a64 100644 --- a/test/tkt-b72787b1.test +++ b/test/tkt-b72787b1.test @@ -35,6 +35,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + unset -nocomplain ::STMT proc runsql {} { db eval {CREATE TABLE IF NOT EXISTS t4(q)} diff --git a/test/tkt-d82e3f3721.test b/test/tkt-d82e3f3721.test index 31f7d342c7..da932d1eaa 100644 --- a/test/tkt-d82e3f3721.test +++ b/test/tkt-d82e3f3721.test @@ -17,6 +17,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt-d82e3-1.1 { db eval { CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); diff --git a/test/tkt-f777251dc7a.test b/test/tkt-f777251dc7a.test index 6f0b43fa8e..25bf667d7d 100644 --- a/test/tkt-f777251dc7a.test +++ b/test/tkt-f777251dc7a.test @@ -17,6 +17,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt-f7772-1.1 { execsql { CREATE TEMP TABLE t1(x UNIQUE); diff --git a/test/tkt3527.test b/test/tkt3527.test index 34e9e6121f..d9b1dadfdc 100644 --- a/test/tkt3527.test +++ b/test/tkt3527.test @@ -18,6 +18,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt3527-1.1 { db eval { CREATE TABLE Element ( diff --git a/test/tkt3773.test b/test/tkt3773.test index 0dc414e50e..3f5a1a3217 100644 --- a/test/tkt3773.test +++ b/test/tkt3773.test @@ -18,6 +18,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !compound { + finish_test + return +} + do_test tkt3773-1.1 { db eval { CREATE TABLE t1(a,b); diff --git a/test/trigger1.test b/test/trigger1.test index 1e77fe0fc8..9d917bdb38 100644 --- a/test/trigger1.test +++ b/test/trigger1.test @@ -29,7 +29,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable {!trigger} { +ifcapable !trigger||!compound { finish_test return } diff --git a/test/vtabD.test b/test/vtabD.test index 509ba45745..589f518983 100644 --- a/test/vtabD.test +++ b/test/vtabD.test @@ -49,17 +49,15 @@ do_test vtabD-1.5 { do_test vtabD-1.6 { execsql { SELECT * FROM tv1 WHERE a < 500 OR b = 810000 } } [execsql { - SELECT * FROM t1 WHERE a < 500 - UNION ALL - SELECT * FROM t1 WHERE b = 810000 AND NOT (a < 500) + SELECT * FROM t1 WHERE a < 500; + SELECT * FROM t1 WHERE b = 810000 AND NOT (a < 500); }] do_test vtabD-1.7 { execsql { SELECT * FROM tv1 WHERE a < 90000 OR b = 8100000000 } } [execsql { - SELECT * FROM t1 WHERE a < 90000 - UNION ALL - SELECT * FROM t1 WHERE b = 8100000000 AND NOT (a < 90000) + SELECT * FROM t1 WHERE a < 90000; + SELECT * FROM t1 WHERE b = 8100000000 AND NOT (a < 90000); }] if {[working_64bit_int]} { diff --git a/test/vtab_shared.test b/test/vtab_shared.test index ce2e432fcf..6a76e2749b 100644 --- a/test/vtab_shared.test +++ b/test/vtab_shared.test @@ -124,23 +124,25 @@ breakpoint execsql { SELECT * FROM t3 } db2 } {1 2 3 4 5 6} -do_test vtab_shared-1.12.1 { - db close - execsql { - SELECT * FROM t1 UNION ALL - SELECT * FROM t2 UNION ALL - SELECT * FROM t3 - } db2 -} {1 2 3 4 5 6 1 2 3 4 5 6 1 2 3 4 5 6} -do_test vtab_shared-1.12.2 { - sqlite3 db test.db - register_echo_module [sqlite3_connection_pointer db] - execsql { - SELECT * FROM t1 UNION ALL - SELECT * FROM t2 UNION ALL - SELECT * FROM t3 - } db -} {1 2 3 4 5 6 1 2 3 4 5 6 1 2 3 4 5 6} +ifcapable compound { + do_test vtab_shared-1.12.1 { + db close + execsql { + SELECT * FROM t1 UNION ALL + SELECT * FROM t2 UNION ALL + SELECT * FROM t3 + } db2 + } {1 2 3 4 5 6 1 2 3 4 5 6 1 2 3 4 5 6} + do_test vtab_shared-1.12.2 { + sqlite3 db test.db + register_echo_module [sqlite3_connection_pointer db] + execsql { + SELECT * FROM t1 UNION ALL + SELECT * FROM t2 UNION ALL + SELECT * FROM t3 + } db + } {1 2 3 4 5 6 1 2 3 4 5 6 1 2 3 4 5 6} +} # Try a rename or two. # diff --git a/test/where9.test b/test/where9.test index b4a2d8d130..8c5c96e2a8 100644 --- a/test/where9.test +++ b/test/where9.test @@ -15,7 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !or_opt { +ifcapable !or_opt||!compound { finish_test return } From 0f198a74092903c3d8d5227454acd055600e1f6e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Feb 2012 16:43:16 +0000 Subject: [PATCH 11/51] Change the ROLLBACK command so that pending statements to not block it. The pending statements instead return SQLITE_ABORT upon next access. Note: This causes separate shared-cache connections in read_uncommitted mode to abort as well. FossilOrigin-Name: 9b66c79e054c8a109d1ee8d35fed1c42814c330d --- manifest | 39 +++++++++++++++++++++------------------ manifest.uuid | 2 +- src/backup.c | 2 +- src/btree.c | 24 ++++++++++-------------- src/btree.h | 2 +- src/main.c | 14 +++++++++----- src/sqliteInt.h | 2 +- src/vdbe.c | 20 +++++++++++--------- src/vdbeaux.c | 39 +++++---------------------------------- test/capi3.test | 13 +++++++------ test/capi3c.test | 13 +++++++------ test/incrblob.test | 10 ++-------- test/savepoint.test | 12 ++++++++++-- test/shared2.test | 42 ------------------------------------------ 14 files changed, 86 insertions(+), 148 deletions(-) diff --git a/manifest b/manifest index bdb31adf3a..03fcacd56d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\svarious\stest\sscripts\sso\sthat\sveryquick.test\sruns\swith\sOMIT_COMPOUND_SELECT\sdefined. -D 2012-02-13T10:00:35.138 +C Change\sthe\sROLLBACK\scommand\sso\sthat\spending\sstatements\sto\snot\sblock\sit.\s\sThe\npending\sstatements\sinstead\sreturn\sSQLITE_ABORT\supon\snext\saccess.\s\sNote:\sThis\ncauses\sseparate\sshared-cache\sconnections\sin\sread_uncommitted\smode\sto\sabort\sas\nwell. +D 2012-02-13T16:43:16.086 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,11 +122,11 @@ F src/alter.c 149cc80d9257971b0bff34e58fb2263e01998289 F src/analyze.c f32ff304da413851eefa562b04e61ff6cb88248b F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c e9538bad2d4a4fcd4308f1aed7cb18a0fbc968f9 +F src/backup.c 6be23a344d3301ae38e92fddb3a33b91c309fce4 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c bb0132a725b4d5ed077924399c4f6d4b9390a721 -F src/btree.h 46e9f04672f1390255bc56865a3238b384d0f2d5 +F src/btree.c 253c3147a4ebbaee42cd329dbdc0856200bbbda7 +F src/btree.h 48a013f8964f12d944d90e4700df47b72dd6d923 F src/btreeInt.h 26d8ca625b141927fe6620c1d2cf58eaf494ca0c F src/build.c c4d36e527f457f9992a6663365871dfa7c5094b8 F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a @@ -147,7 +147,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d -F src/main.c cb099cc4864b542f97938049fc74ea8ae6845ce4 +F src/main.c 4aa7f1994a7cf810d7d754cb30202543b40bb726 F src/malloc.c 15afac5e59b6584efe072e9933aefb4230e74f97 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c b3677415e69603d6a0e7c5410a1b3731d55beda1 @@ -184,7 +184,7 @@ F src/select.c 232283a2e60d91cbd9a5ddf2f6f7ecf53d590075 F src/shell.c aa28f117033ba3e44b5eaaf2ad572222bcdfd66e F src/sqlite.h.in 371c30e4be94b9b0ea6400ed66663fcf8e891eb4 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h 736f3a7748434200db7a9e0a352b67d1b8759e0a +F src/sqliteInt.h ea24b5f2d2e3ae7e0c1126a78c0e025ed4e98f4b F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 4568e72dfd36b6a5911f93457364deb072e0b03a F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -240,11 +240,11 @@ F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 9e07bd67dfafe9c75b1da78c87ba030cebbb5388 F src/vacuum.c 0c0ba2242355c6048d65e2b333abe0f7c06348fa -F src/vdbe.c 40b14dff04692b1ee421db40c67d4921ecf17a9d +F src/vdbe.c 2afc9b0260dfe47786211e1eb815a52f27c661b7 F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 F src/vdbeapi.c 3662b6a468a2a4605a15dfab313baa6dff81ad91 -F src/vdbeaux.c 7683d772ad638faa4567142438c4594e47f173c4 +F src/vdbeaux.c dce061e42f89cd6d0af15c03a854bcc89bd5deb0 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c fb0ac964ccbcd94f595eb993c05bfd9c52468a4a F src/vdbesort.c b25814d385895544ebc8118245c8311ded7f81c9 @@ -314,9 +314,9 @@ F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test f64136b0893c293d0b910ed057b3b711249099a7 F test/capi2.test 835d4cee9f542ea50fa8d01f3fe6de80b0627360 -F test/capi3.test 7200dff6acb17b9a4b6f9918f554eaae04968ddd +F test/capi3.test 9c8b58b6a6aeb14e69bd8c8c7721b47d640464d1 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test ccf0acf045dbacd09f6229aa4efed670aaba76a9 +F test/capi3c.test 1b5424d2ac57b7b443b5de5b9a287642c02279b6 F test/capi3d.test 17b57ca28be3e37e14c2ba8f787d292d84b724a1 F test/capi3e.test f7408dda65c92b9056199fdc180f893015f83dde F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 @@ -511,7 +511,7 @@ F test/in.test a7b8a0f43da81cd08645b7a710099ffe9ad1126b F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 -F test/incrblob.test 3307c04876fe025e10256e3cc8050ab5a84aa27f +F test/incrblob.test 26fde912a1e0aff158b3a84ef3b265f046aad3be F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob_err.test d2562d2771ebffd4b3af89ef64c140dd44371597 @@ -654,7 +654,7 @@ F test/rollback.test a1b4784b864331eae8b2a98c189efa2a8b11ff07 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test e58e0acef38b527ed1b0b70d3ada588f804af287 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 -F test/savepoint.test 42ba63b632df4ce1bef8acdba62e4182d6505125 +F test/savepoint.test f5acd87d0c7a5f4ad6c547b47fd18c0e1aeaf048 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 @@ -679,7 +679,7 @@ F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 F test/shared.test 34945a516532b11182c3eb26e31247eee3c9ae48 -F test/shared2.test 8f71d4eb4d5261280de92284df74172545c852cc +F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 F test/shared3.test ebf77f023f4bdaa8f74f65822b559e86ce5c6257 F test/shared4.test 72d90821e8d2fc918a08f16d32880868d8ee8e9d F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9 @@ -989,7 +989,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P bfbfe05b81919ecc3d6e7be4c24994f795f16582 -R 6250e4472fb78736176921352e48e2f9 -U dan -Z f0319cf74449ae83da7435ab8330767c +P 76bb649ee2633226324130f5898622c348f93769 +R 99d7e91688392c96521a36db0be702a8 +T *branch * nonblocking-rollback +T *sym-nonblocking-rollback * +T -sym-trunk * +U drh +Z 742e7385f2147385b0b62b5bc1ba4557 diff --git a/manifest.uuid b/manifest.uuid index 896ec58a38..92a77a8c19 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -76bb649ee2633226324130f5898622c348f93769 \ No newline at end of file +9b66c79e054c8a109d1ee8d35fed1c42814c330d \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index aa3401897b..7a4047f34c 100644 --- a/src/backup.c +++ b/src/backup.c @@ -568,7 +568,7 @@ int sqlite3_backup_finish(sqlite3_backup *p){ } /* If a transaction is still open on the Btree, roll it back. */ - sqlite3BtreeRollback(p->pDest); + sqlite3BtreeRollback(p->pDest, SQLITE_OK); /* Set the error code of the destination database handle. */ rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; diff --git a/src/btree.c b/src/btree.c index 359253387d..68345144a8 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2041,7 +2041,7 @@ int sqlite3BtreeClose(Btree *p){ ** The call to sqlite3BtreeRollback() drops any table-locks held by ** this handle. */ - sqlite3BtreeRollback(p); + sqlite3BtreeRollback(p, SQLITE_OK); sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree @@ -3279,6 +3279,7 @@ static int countWriteCursors(BtShared *pBt){ */ void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ BtCursor *p; + if( pBtree==0 ) return; sqlite3BtreeEnter(pBtree); for(p=pBtree->pBt->pCursor; p; p=p->pNext){ int i; @@ -3302,25 +3303,20 @@ void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ ** This will release the write lock on the database file. If there ** are no active cursors, it also releases the read lock. */ -int sqlite3BtreeRollback(Btree *p){ +int sqlite3BtreeRollback(Btree *p, int tripCode){ int rc; BtShared *pBt = p->pBt; MemPage *pPage1; sqlite3BtreeEnter(p); - rc = saveAllCursors(pBt, 0, 0); -#ifndef SQLITE_OMIT_SHARED_CACHE - if( rc!=SQLITE_OK ){ - /* This is a horrible situation. An IO or malloc() error occurred whilst - ** trying to save cursor positions. If this is an automatic rollback (as - ** the result of a constraint, malloc() failure or IO error) then - ** the cache may be internally inconsistent (not contain valid trees) so - ** we cannot simply return the error to the caller. Instead, abort - ** all queries that may be using any of the cursors that failed to save. - */ - sqlite3BtreeTripAllCursors(p, rc); + if( tripCode==SQLITE_OK ){ + rc = tripCode = saveAllCursors(pBt, 0, 0); + }else{ + rc = SQLITE_OK; + } + if( tripCode ){ + sqlite3BtreeTripAllCursors(p, tripCode); } -#endif btreeIntegrity(p); if( p->inTrans==TRANS_WRITE ){ diff --git a/src/btree.h b/src/btree.h index 637dc486e1..9832001b7f 100644 --- a/src/btree.h +++ b/src/btree.h @@ -77,7 +77,7 @@ int sqlite3BtreeBeginTrans(Btree*,int); int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); int sqlite3BtreeCommitPhaseTwo(Btree*, int); int sqlite3BtreeCommit(Btree*); -int sqlite3BtreeRollback(Btree*); +int sqlite3BtreeRollback(Btree*,int); int sqlite3BtreeBeginStmt(Btree*,int); int sqlite3BtreeCreateTable(Btree*, int*, int flags); int sqlite3BtreeIsInTrans(Btree*); diff --git a/src/main.c b/src/main.c index 8838a0356f..21a40842ca 100644 --- a/src/main.c +++ b/src/main.c @@ -849,19 +849,23 @@ int sqlite3_close(sqlite3 *db){ } /* -** Rollback all database files. +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any open cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. */ -void sqlite3RollbackAll(sqlite3 *db){ +void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int i; int inTrans = 0; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); for(i=0; inDb; i++){ - if( db->aDb[i].pBt ){ - if( sqlite3BtreeIsInTrans(db->aDb[i].pBt) ){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeIsInTrans(p) ){ inTrans = 1; } - sqlite3BtreeRollback(db->aDb[i].pBt); + sqlite3BtreeRollback(p, tripCode); db->aDb[i].inTrans = 0; } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 58a6450783..8165d05bff 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2800,7 +2800,7 @@ Vdbe *sqlite3GetVdbe(Parse*); void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); void sqlite3PrngResetState(void); -void sqlite3RollbackAll(sqlite3*); +void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); void sqlite3BeginTransaction(Parse*, int); diff --git a/src/vdbe.c b/src/vdbe.c index 9ce25942dd..fbbaf1af34 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2684,16 +2684,12 @@ case OP_Savepoint: { if( !pSavepoint ){ sqlite3SetString(&p->zErrMsg, db, "no such savepoint: %s", zName); rc = SQLITE_ERROR; - }else if( - db->writeVdbeCnt>0 || (p1==SAVEPOINT_ROLLBACK && db->activeVdbeCnt>1) - ){ + }else if( db->writeVdbeCnt>0 && p1==SAVEPOINT_RELEASE ){ /* It is not possible to release (commit) a savepoint if there are - ** active write statements. It is not possible to rollback a savepoint - ** if there are any active statements at all. + ** active write statements. */ sqlite3SetString(&p->zErrMsg, db, - "cannot %s savepoint - SQL statements in progress", - (p1==SAVEPOINT_ROLLBACK ? "rollback": "release") + "cannot release savepoint - SQL statements in progress" ); rc = SQLITE_BUSY; }else{ @@ -2718,6 +2714,9 @@ case OP_Savepoint: { rc = p->rc; }else{ iSavepoint = db->nSavepoint - iSavepoint - 1; + for(ii=0; iinDb; ii++){ + sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + } for(ii=0; iinDb; ii++){ rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); if( rc!=SQLITE_OK ){ @@ -2786,6 +2785,7 @@ case OP_AutoCommit: { assert( desiredAutoCommit==1 || iRollback==0 ); assert( db->activeVdbeCnt>0 ); /* At least this one VM is active */ +#if 0 if( turnOnAC && iRollback && db->activeVdbeCnt>1 ){ /* If this instruction implements a ROLLBACK and other VMs are ** still running, and a transaction is active, return an error indicating @@ -2794,7 +2794,9 @@ case OP_AutoCommit: { sqlite3SetString(&p->zErrMsg, db, "cannot rollback transaction - " "SQL statements in progress"); rc = SQLITE_BUSY; - }else if( turnOnAC && !iRollback && db->writeVdbeCnt>0 ){ + }else +#endif + if( turnOnAC && !iRollback && db->writeVdbeCnt>0 ){ /* If this instruction implements a COMMIT and other VMs are writing ** return an error indicating that the other VMs must complete first. */ @@ -2804,7 +2806,7 @@ case OP_AutoCommit: { }else if( desiredAutoCommit!=db->autoCommit ){ if( iRollback ){ assert( desiredAutoCommit==1 ); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT); db->autoCommit = 1; }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 9b54cb6bec..c85a635891 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2002,32 +2002,6 @@ static void checkActiveVdbeCnt(sqlite3 *db){ #define checkActiveVdbeCnt(x) #endif -/* -** For every Btree that in database connection db which -** has been modified, "trip" or invalidate each cursor in -** that Btree might have been modified so that the cursor -** can never be used again. This happens when a rollback -*** occurs. We have to trip all the other cursors, even -** cursor from other VMs in different database connections, -** so that none of them try to use the data at which they -** were pointing and which now may have been changed due -** to the rollback. -** -** Remember that a rollback can delete tables complete and -** reorder rootpages. So it is not sufficient just to save -** the state of the cursor. We have to invalidate the cursor -** so that it is never used again. -*/ -static void invalidateCursorsOnModifiedBtrees(sqlite3 *db){ - int i; - for(i=0; inDb; i++){ - Btree *p = db->aDb[i].pBt; - if( p && sqlite3BtreeIsInTrans(p) ){ - sqlite3BtreeTripAllCursors(p, SQLITE_ABORT); - } - } -} - /* ** If the Vdbe passed as the first argument opened a statement-transaction, ** close it now. Argument eOp must be either SAVEPOINT_ROLLBACK or @@ -2192,8 +2166,7 @@ int sqlite3VdbeHalt(Vdbe *p){ /* We are forced to roll back the active transaction. Before doing ** so, abort any other statements this handle currently has active. */ - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT); sqlite3CloseSavepoints(db); db->autoCommit = 1; } @@ -2235,13 +2208,13 @@ int sqlite3VdbeHalt(Vdbe *p){ return SQLITE_BUSY; }else if( rc!=SQLITE_OK ){ p->rc = rc; - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_OK); }else{ db->nDeferredCons = 0; sqlite3CommitInternalChanges(db); } }else{ - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_OK); } db->nStatement = 0; }else if( eStatementOp==0 ){ @@ -2250,8 +2223,7 @@ int sqlite3VdbeHalt(Vdbe *p){ }else if( p->errorAction==OE_Abort ){ eStatementOp = SAVEPOINT_ROLLBACK; }else{ - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT); sqlite3CloseSavepoints(db); db->autoCommit = 1; } @@ -2271,8 +2243,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } - invalidateCursorsOnModifiedBtrees(db); - sqlite3RollbackAll(db); + sqlite3RollbackAll(db, SQLITE_ABORT); sqlite3CloseSavepoints(db); db->autoCommit = 1; } diff --git a/test/capi3.test b/test/capi3.test index cbba04cd84..a9aab1e615 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -894,18 +894,19 @@ do_test capi3-11.9.2 { catchsql { ROLLBACK; } -} {1 {cannot rollback transaction - SQL statements in progress}} +} {0 {}} do_test capi3-11.9.3 { sqlite3_get_autocommit $DB -} 0 +} 1 do_test capi3-11.10 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_ERROR} do_test capi3-11.11 { sqlite3_step $STMT } {SQLITE_ROW} do_test capi3-11.12 { sqlite3_step $STMT + sqlite3_step $STMT } {SQLITE_DONE} do_test capi3-11.13 { sqlite3_finalize $STMT @@ -914,15 +915,15 @@ do_test capi3-11.14 { execsql { SELECT a FROM t2; } -} {1 2 3} +} {1 2} do_test capi3-11.14.1 { sqlite3_get_autocommit $DB -} 0 +} 1 do_test capi3-11.15 { catchsql { ROLLBACK; } -} {0 {}} +} {1 {cannot rollback - no transaction is active}} do_test capi3-11.15.1 { sqlite3_get_autocommit $DB } 1 diff --git a/test/capi3c.test b/test/capi3c.test index c1d5510bd8..adef7f9189 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -849,18 +849,19 @@ do_test capi3c-11.9.2 { catchsql { ROLLBACK; } -} {1 {cannot rollback transaction - SQL statements in progress}} +} {0 {}} do_test capi3c-11.9.3 { sqlite3_get_autocommit $DB -} 0 +} 1 do_test capi3c-11.10 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_ABORT} do_test capi3c-11.11 { sqlite3_step $STMT } {SQLITE_ROW} do_test capi3c-11.12 { sqlite3_step $STMT + sqlite3_step $STMT } {SQLITE_DONE} do_test capi3c-11.13 { sqlite3_finalize $STMT @@ -869,15 +870,15 @@ do_test capi3c-11.14 { execsql { SELECT a FROM t2; } -} {1 2 3} +} {1 2} do_test capi3c-11.14.1 { sqlite3_get_autocommit $DB -} 0 +} 1 do_test capi3c-11.15 { catchsql { ROLLBACK; } -} {0 {}} +} {1 {cannot rollback - no transaction is active}} do_test capi3c-11.15.1 { sqlite3_get_autocommit $DB } 1 diff --git a/test/incrblob.test b/test/incrblob.test index 388c4ba3a3..1880128f83 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -473,15 +473,9 @@ if {[permutation] != "memsubsys1"} { flush $::blob } {} - # At this point rollback should be illegal (because - # there is an open blob channel). But commit is also illegal because - # the open blob is read-write. + # At this point commit should be illegal (because + # there is an open blob channel). # - do_test incrblob-6.10 { - catchsql { - ROLLBACK; - } db2 - } {1 {cannot rollback transaction - SQL statements in progress}} do_test incrblob-6.11 { catchsql { COMMIT; diff --git a/test/savepoint.test b/test/savepoint.test index 6163e2e8a9..015d97fe1a 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -303,11 +303,19 @@ ifcapable incrblob { execsql {SAVEPOINT abc} catchsql {ROLLBACK TO def} } {1 {no such savepoint: def}} - do_test savepoint-5.3.2 { + do_test savepoint-5.3.2.1 { execsql {SAVEPOINT def} set fd [db incrblob -readonly blobs x 1] + set rc [catch {seek $fd 0;read $fd} res] + lappend rc $res + } {0 {hellontyeight character blob}} + do_test savepoint-5.3.2.2 { catchsql {ROLLBACK TO def} - } {1 {cannot rollback savepoint - SQL statements in progress}} + } {0 {}} + do_test savepoint-5.3.2.3 { + set rc [catch {seek $fd 0; read $fd} res] + set rc + } {1} do_test savepoint-5.3.3 { catchsql {RELEASE def} } {0 {}} diff --git a/test/shared2.test b/test/shared2.test index d40c9a271a..5bde8cfad8 100644 --- a/test/shared2.test +++ b/test/shared2.test @@ -79,48 +79,6 @@ do_test shared2-1.3 { list $a $count } {32 64} -#--------------------------------------------------------------------------- -# These tests, shared2.2.*, test the outcome when data is added to or -# removed from a table due to a rollback while a read-uncommitted -# cursor is scanning it. -# -do_test shared2-2.1 { - execsql { - INSERT INTO numbers VALUES(1, 'Medium length text field'); - INSERT INTO numbers VALUES(2, 'Medium length text field'); - INSERT INTO numbers VALUES(3, 'Medium length text field'); - INSERT INTO numbers VALUES(4, 'Medium length text field'); - BEGIN; - DELETE FROM numbers WHERE (a%2)=0; - } db1 - set res [list] - db2 eval { - SELECT a FROM numbers ORDER BY a; - } { - lappend res $a - if {$a==3} { - execsql {ROLLBACK} db1 - } - } - set res -} {1 3 4} -do_test shared2-2.2 { - execsql { - BEGIN; - INSERT INTO numbers VALUES(5, 'Medium length text field'); - INSERT INTO numbers VALUES(6, 'Medium length text field'); - } db1 - set res [list] - db2 eval { - SELECT a FROM numbers ORDER BY a; - } { - lappend res $a - if {$a==5} { - execsql {ROLLBACK} db1 - } - } - set res -} {1 2 3 4 5} db1 close db2 close From 21021a5cbfdd904573412ee72908dbd47f37de4a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Feb 2012 17:01:51 +0000 Subject: [PATCH 12/51] Add the new SQLITE_ABORT_ROLLBACK extended error code to be returned for statements that are cancelled due to a rollback. FossilOrigin-Name: 549f4fd00d8325c10099b100e5202b77ee1d83ad --- manifest | 25 +++++++++++-------------- manifest.uuid | 2 +- src/main.c | 19 ++++++++++++++----- src/sqlite.h.in | 1 + src/vdbe.c | 2 +- src/vdbeaux.c | 6 +++--- test/tkt-f777251dc7a.test | 2 +- test/trans3.test | 7 +++---- 8 files changed, 35 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 03fcacd56d..24d2878880 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sROLLBACK\scommand\sso\sthat\spending\sstatements\sto\snot\sblock\sit.\s\sThe\npending\sstatements\sinstead\sreturn\sSQLITE_ABORT\supon\snext\saccess.\s\sNote:\sThis\ncauses\sseparate\sshared-cache\sconnections\sin\sread_uncommitted\smode\sto\sabort\sas\nwell. -D 2012-02-13T16:43:16.086 +C Add\sthe\snew\sSQLITE_ABORT_ROLLBACK\sextended\serror\scode\sto\sbe\sreturned\sfor\nstatements\sthat\sare\scancelled\sdue\sto\sa\srollback. +D 2012-02-13T17:01:51.864 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -147,7 +147,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d -F src/main.c 4aa7f1994a7cf810d7d754cb30202543b40bb726 +F src/main.c e936269fbc1af78cef79c27980e89209aeb20a07 F src/malloc.c 15afac5e59b6584efe072e9933aefb4230e74f97 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c b3677415e69603d6a0e7c5410a1b3731d55beda1 @@ -182,7 +182,7 @@ F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 232283a2e60d91cbd9a5ddf2f6f7ecf53d590075 F src/shell.c aa28f117033ba3e44b5eaaf2ad572222bcdfd66e -F src/sqlite.h.in 371c30e4be94b9b0ea6400ed66663fcf8e891eb4 +F src/sqlite.h.in 4f3230be44e084a1e98f91bb031b922f33dcf3f8 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 F src/sqliteInt.h ea24b5f2d2e3ae7e0c1126a78c0e025ed4e98f4b F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d @@ -240,11 +240,11 @@ F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 9e07bd67dfafe9c75b1da78c87ba030cebbb5388 F src/vacuum.c 0c0ba2242355c6048d65e2b333abe0f7c06348fa -F src/vdbe.c 2afc9b0260dfe47786211e1eb815a52f27c661b7 +F src/vdbe.c 21c17c43e579789959efea50f100de60efbb955c F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 F src/vdbeapi.c 3662b6a468a2a4605a15dfab313baa6dff81ad91 -F src/vdbeaux.c dce061e42f89cd6d0af15c03a854bcc89bd5deb0 +F src/vdbeaux.c 79cf42b70e211a52d664fc4d585ee2da0a64deac F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c fb0ac964ccbcd94f595eb993c05bfd9c52468a4a F src/vdbesort.c b25814d385895544ebc8118245c8311ded7f81c9 @@ -763,7 +763,7 @@ F test/tkt-d11f09d36e.test fb44f7961aa6d4b632fb7b9768239832210b5fc7 F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test 669bb076f2ac573c7398ce00f40cd0ca502043a9 -F test/tkt-f777251dc7a.test a3121f92f49af447572f7a3cde9f07806443c75d +F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 1da0ed15ec2c7749fb5ce2828cd69d07153ad9f4 F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 @@ -856,7 +856,7 @@ F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 F test/trace2.test 962175290996d5f06dc4402ca218bbfc7df4cb20 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 -F test/trans3.test d728abaa318ca364dc370e06576aa7e5fbed7e97 +F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 F test/trigger1.test de42feb7cd442787d38185ae74f5a1d7afa400cb F test/trigger2.test 834187beafd1db383af0c659cfa49b0576832816 F test/trigger3.test d2c60d8be271c355d61727411e753181e877230a @@ -989,10 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 76bb649ee2633226324130f5898622c348f93769 -R 99d7e91688392c96521a36db0be702a8 -T *branch * nonblocking-rollback -T *sym-nonblocking-rollback * -T -sym-trunk * +P 9b66c79e054c8a109d1ee8d35fed1c42814c330d +R 1066e7f5b671895741b8f4418594178f U drh -Z 742e7385f2147385b0b62b5bc1ba4557 +Z 81fc65771654ca843df55065930e81f0 diff --git a/manifest.uuid b/manifest.uuid index 92a77a8c19..67d01dda7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b66c79e054c8a109d1ee8d35fed1c42814c330d \ No newline at end of file +549f4fd00d8325c10099b100e5202b77ee1d83ad \ No newline at end of file diff --git a/src/main.c b/src/main.c index 21a40842ca..dedfba4a19 100644 --- a/src/main.c +++ b/src/main.c @@ -920,12 +920,21 @@ const char *sqlite3ErrStr(int rc){ /* SQLITE_RANGE */ "bind or column index out of range", /* SQLITE_NOTADB */ "file is encrypted or is not a database", }; - rc &= 0xff; - if( ALWAYS(rc>=0) && rc<(int)(sizeof(aMsg)/sizeof(aMsg[0])) && aMsg[rc]!=0 ){ - return aMsg[rc]; - }else{ - return "unknown error"; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rcautoCommit ){ if( iRollback ){ assert( desiredAutoCommit==1 ); - sqlite3RollbackAll(db, SQLITE_ABORT); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); db->autoCommit = 1; }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index c85a635891..ef339d1e13 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2166,7 +2166,7 @@ int sqlite3VdbeHalt(Vdbe *p){ /* We are forced to roll back the active transaction. Before doing ** so, abort any other statements this handle currently has active. */ - sqlite3RollbackAll(db, SQLITE_ABORT); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; } @@ -2223,7 +2223,7 @@ int sqlite3VdbeHalt(Vdbe *p){ }else if( p->errorAction==OE_Abort ){ eStatementOp = SAVEPOINT_ROLLBACK; }else{ - sqlite3RollbackAll(db, SQLITE_ABORT); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; } @@ -2243,7 +2243,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } - sqlite3RollbackAll(db, SQLITE_ABORT); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; } diff --git a/test/tkt-f777251dc7a.test b/test/tkt-f777251dc7a.test index 25bf667d7d..af6f71ad96 100644 --- a/test/tkt-f777251dc7a.test +++ b/test/tkt-f777251dc7a.test @@ -42,7 +42,7 @@ do_test tkt-f7772-1.2 { BEGIN IMMEDIATE; SELECT x, force_rollback(), EXISTS(SELECT 1 FROM t3 WHERE w=x) FROM t2; } -} {1 {callback requested query abort}} +} {1 {abort due to ROLLBACK}} do_test tkt-f7772-1.3 { sqlite3_get_autocommit db } {1} diff --git a/test/trans3.test b/test/trans3.test index ab7db6a277..d5b316bec8 100644 --- a/test/trans3.test +++ b/test/trans3.test @@ -64,14 +64,13 @@ do_test trans3-1.5 { } } errmsg] lappend x $errmsg -} {1 {cannot rollback transaction - SQL statements in progress}} +} {1 {abort due to ROLLBACK}} do_test trans3-1.6 { set ::ecode -} {SQLITE_BUSY} +} {} do_test trans3-1.7 { - db eval COMMIT db eval {SELECT * FROM t1} -} {1 2 3 4 5} +} {1 2 3 4} unset -nocomplain ecode finish_test From 8c815d14f88c68285508fbc4001a4cd1a0dd783b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Feb 2012 20:16:37 +0000 Subject: [PATCH 13/51] On unix, ignore the umask when creating journal files. That way, journal files will have exactly the same permissions as the original database and any process that has permission to write to the database can also recover hot journals. FossilOrigin-Name: 84b324606adc8437338c086404eb157f30f04130 --- manifest | 20 +++++++-------- manifest.uuid | 2 +- src/os_unix.c | 63 +++++++++++++++++++++++++++++++--------------- test/journal3.test | 7 ++++-- test/syscall.test | 2 +- test/wal2.test | 5 +++- 6 files changed, 64 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index bdb31adf3a..014ff3285d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\svarious\stest\sscripts\sso\sthat\sveryquick.test\sruns\swith\sOMIT_COMPOUND_SELECT\sdefined. -D 2012-02-13T10:00:35.138 +C On\sunix,\signore\sthe\sumask\swhen\screating\sjournal\sfiles.\s\sThat\sway,\sjournal\sfiles\nwill\shave\sexactly\sthe\ssame\spermissions\sas\sthe\soriginal\sdatabase\sand\sany\nprocess\sthat\shas\spermission\sto\swrite\sto\sthe\sdatabase\scan\salso\srecover\nhot\sjournals. +D 2012-02-13T20:16:37.535 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c fd5875e602286c8ecd96a0049bd7869c30721df0 +F src/os_unix.c 8035d8af66d205b6bfd249c1ed0eb708c8411a3c F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a @@ -548,7 +548,7 @@ F test/join5.test 86675fc2919269aa923c84dd00ee4249b97990fe F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 8b71ef1ed5798bdc0e6eb616d8694e2c2c188d4d F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 -F test/journal3.test 6fd28532c88b447db844186bc190523108b6dbb4 +F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 9ee3a78f53d52cca737db69293d15dc41c0cbd36 F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa @@ -709,7 +709,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test 8a1bd9575ea1e8bdc0513fc5be7753fa4c3c04fb +F test/syscall.test bea9bf329bff733c791310244617c2a76974e64a F test/sysfault.test c79441d88d23696fbec7b147dba98d42a04f523f F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 @@ -906,7 +906,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 F test/wal.test edefe316b4125d7f68004ea53c5e73c398d436cc -F test/wal2.test f11883dd3cb7f647c5d2acfd7b5c6d4ba5770cc9 +F test/wal2.test 8871e7fd2c86711ff415a5817d68ea3101a15312 F test/wal3.test 6504bbf348b2d6dfade64a064f1050fd617e8706 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test f58ed4b8b542f71c7441da12fbd769d99b362437 @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P bfbfe05b81919ecc3d6e7be4c24994f795f16582 -R 6250e4472fb78736176921352e48e2f9 -U dan -Z f0319cf74449ae83da7435ab8330767c +P 76bb649ee2633226324130f5898622c348f93769 +R b5d60671b244f9108cc239280d062aa2 +U drh +Z 7eb3e8117d2ffa2020d44f7552f02a09 diff --git a/manifest.uuid b/manifest.uuid index 896ec58a38..b443c6e1a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -76bb649ee2633226324130f5898622c348f93769 \ No newline at end of file +84b324606adc8437338c086404eb157f30f04130 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 31fb1b40d9..ef6cba910a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -422,6 +422,9 @@ static struct unix_syscall { { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + { "umask", (sqlite3_syscall_ptr)umask, 0 }, +#define osUmask ((mode_t(*)(mode_t))aSyscall[21].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -508,11 +511,36 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ } /* -** Retry open() calls that fail due to EINTR +** Invoke open(). Do so multiple times, until it either succeeds or +** files for some reason other than EINTR. +** +** If the file creation mode "m" is 0 then set it to the default for +** SQLite. The default is SQLITE_DEFAULT_FILE_PERMISSIONS (normally +** 0644) as modified by the system umask. If m is not 0, then +** make the file creation mode be exactly m ignoring the umask. +** +** The m parameter will be non-zero only when creating -wal, -journal, +** and -shm files. We want those files to have *exactly* the same +** permissions as their original database, unadulterated by the umask. +** In that way, if a database file is -rw-rw-rw or -rw-rw-r-, and a +** transaction crashes and leaves behind hot journals, then any +** process that is able to write to the database will also be able to +** recover the hot journals. */ -static int robust_open(const char *z, int f, int m){ +static int robust_open(const char *z, int f, mode_t m){ int rc; - do{ rc = osOpen(z,f,m); }while( rc<0 && errno==EINTR ); + mode_t m2; + mode_t origM; + if( m==0 ){ + m2 = SQLITE_DEFAULT_FILE_PERMISSIONS; + }else{ + m2 = m; + origM = osUmask(0); + } + do{ rc = osOpen(z,f,m2); }while( rc<0 && errno==EINTR ); + if( m ){ + osUmask(origM); + } return rc; } @@ -3860,8 +3888,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ /* Call fstat() to figure out the permissions on the database file. If ** a new *-shm file is created, an attempt will be made to create it - ** with the same permissions. The actual permissions the file is created - ** with are subject to the current umask setting. + ** with the same permissions. */ if( osFstat(pDbFd->h, &sStat) && pInode->bProcessLock==0 ){ rc = SQLITE_IOERR_FSTAT; @@ -4892,12 +4919,10 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** written to *pMode. If an IO error occurs, an SQLite error code is ** returned and the value of *pMode is not modified. ** -** If the file being opened is a temporary file, it is always created with -** the octal permissions 0600 (read/writable by owner only). If the file -** is a database or master journal file, it is created with the permissions -** mask SQLITE_DEFAULT_FILE_PERMISSIONS. -** -** Finally, if the file being opened is a WAL or regular journal file, then +** In most cases cases, this routine sets *pMode to 0, which will become +** an indication to robust_open() to create the file using +** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. +** But if the file being opened is a WAL or regular journal file, then ** this function queries the file-system for the permissions on the ** corresponding database file and sets *pMode to this value. Whenever ** possible, WAL and journal files are created using the same permissions @@ -4916,7 +4941,7 @@ static int findCreateFileMode( gid_t *pGid /* OUT: gid to set on the file */ ){ int rc = SQLITE_OK; /* Return Code */ - *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; + *pMode = 0; *pUid = 0; *pGid = 0; if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ @@ -5860,17 +5885,17 @@ static int proxyCreateUnixFile( } } if( fd<0 ){ - fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(path, openFlags, 0); terrno = errno; if( fd<0 && errno==ENOENT && islockfile ){ if( proxyCreateLockPath(path) == SQLITE_OK ){ - fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(path, openFlags, 0); } } } if( fd<0 ){ openFlags = O_RDONLY; - fd = robust_open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(path, openFlags, 0); terrno = errno; } if( fd<0 ){ @@ -5994,8 +6019,7 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ goto end_breaklock; } /* write it out to the temporary break file */ - fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL), - SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL), 0); if( fd<0 ){ sqlite3_snprintf(sizeof(errmsg), errmsg, "create failed (%d)", errno); goto end_breaklock; @@ -6272,8 +6296,7 @@ static int proxyTakeConch(unixFile *pFile){ robust_close(pFile, pFile->h, __LINE__); } pFile->h = -1; - fd = robust_open(pCtx->dbPath, pFile->openFlags, - SQLITE_DEFAULT_FILE_PERMISSIONS); + fd = robust_open(pCtx->dbPath, pFile->openFlags, 0); OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); if( fd>=0 ){ pFile->h = fd; @@ -6842,7 +6865,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==21 ); + assert( ArraySize(aSyscall)==22 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/test/journal3.test b/test/journal3.test index f1bf89f8c7..939cc27c70 100644 --- a/test/journal3.test +++ b/test/journal3.test @@ -22,7 +22,9 @@ source $testdir/malloc_common.tcl # if {$::tcl_platform(platform) == "unix"} { - set umask [exec /bin/sh -c umask] + # Changed on 2012-02-13: umask is deliberately ignored for -wal, -journal, + # and -shm files. + #set umask [exec /bin/sh -c umask] faultsim_delete_and_reopen do_test journal3-1.1 { execsql { CREATE TABLE tx(y, z) } } {} @@ -33,7 +35,8 @@ if {$::tcl_platform(platform) == "unix"} { 4 00755 } { db close - set effective [format %.5o [expr $permissions & ~$umask]] + #set effective [format %.5o [expr $permissions & ~$umask]] + set effective $permissions do_test journal3-1.2.$tn.1 { catch { forcedelete test.db-journal } file attributes test.db -permissions $permissions diff --git a/test/syscall.test b/test/syscall.test index da7ecd53ba..d841a9a0b2 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -60,7 +60,7 @@ foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir - statvfs fchown + statvfs fchown umask } { if {[test_syscall exists $s]} {lappend syscall_list $s} } diff --git a/test/wal2.test b/test/wal2.test index c7f00ea60f..cbefb7ab1e 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -1042,7 +1042,10 @@ tvfs delete # if {$::tcl_platform(platform) == "unix"} { faultsim_delete_and_reopen - set umask [exec /bin/sh -c umask] + # Changed on 2012-02-13: umask is deliberately ignored for -wal files. + #set umask [exec /bin/sh -c umask] + set umask 0 + do_test wal2-12.1 { sqlite3 db test.db From ef595984ec305eb6cb5f4495ddafcdcef1b3981f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Feb 2012 20:28:15 +0000 Subject: [PATCH 14/51] Fix a harmless compiler warning introduced by the previous check-in. FossilOrigin-Name: a8a042a751557d06bf04455ed7629cb29adcd87f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 014ff3285d..9583d49b64 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C On\sunix,\signore\sthe\sumask\swhen\screating\sjournal\sfiles.\s\sThat\sway,\sjournal\sfiles\nwill\shave\sexactly\sthe\ssame\spermissions\sas\sthe\soriginal\sdatabase\sand\sany\nprocess\sthat\shas\spermission\sto\swrite\sto\sthe\sdatabase\scan\salso\srecover\nhot\sjournals. -D 2012-02-13T20:16:37.535 +C Fix\sa\sharmless\scompiler\swarning\sintroduced\sby\sthe\sprevious\scheck-in. +D 2012-02-13T20:28:15.804 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 8035d8af66d205b6bfd249c1ed0eb708c8411a3c +F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 76bb649ee2633226324130f5898622c348f93769 -R b5d60671b244f9108cc239280d062aa2 +P 84b324606adc8437338c086404eb157f30f04130 +R d3286eff693659b9b4732d3698634dbe U drh -Z 7eb3e8117d2ffa2020d44f7552f02a09 +Z ade107925d4b7260ea92f951a93b4e84 diff --git a/manifest.uuid b/manifest.uuid index b443c6e1a9..0fa6079cc8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84b324606adc8437338c086404eb157f30f04130 \ No newline at end of file +a8a042a751557d06bf04455ed7629cb29adcd87f \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index ef6cba910a..48c130935e 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -530,7 +530,7 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ static int robust_open(const char *z, int f, mode_t m){ int rc; mode_t m2; - mode_t origM; + mode_t origM = 0; if( m==0 ){ m2 = SQLITE_DEFAULT_FILE_PERMISSIONS; }else{ From 5beafd6ab7647663a1c601bb47dc3af22f6b77d2 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Feb 2012 15:34:50 +0000 Subject: [PATCH 15/51] Enhance the fuzzer virtual table to support multiple rule sets. FossilOrigin-Name: a82938731b21d6166d7d482994cb065c8b725083 --- manifest | 14 ++-- manifest.uuid | 2 +- src/test_fuzzer.c | 166 +++++++++++++++++++++++++++++++++++----------- test/fuzzer1.test | 58 ++++++++++++++++ 4 files changed, 192 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index e43e6ab96e..d92a333bed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\snon-blocking\sROLLBACK\schanges\sinto\strunk. -D 2012-02-13T21:24:03.262 +C Enhance\sthe\sfuzzer\svirtual\stable\sto\ssupport\smultiple\srule\ssets. +D 2012-02-14T15:34:50.192 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/test_config.c a036a69b550ebc477ab9ca2b37269201f888436e F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 6232d722a4ddb193035aa13a03796bf57d6c12fd -F src/test_fuzzer.c f884f6f32e8513d34248d6e1ac8a32047fead254 +F src/test_fuzzer.c 0b11b466bd9c5dc8d882d29bb8c7e576963fd905 F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -504,7 +504,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzer1.test ddfb04f3bd5cfdda3b1aa15b78d3ad055c9cc50f +F test/fuzzer1.test ff725a0eec070dfc2b2acc13b21a52a139382929 F test/hook.test 5f3749de6462a6b87b4209b74adf7df5ac2df639 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test a7b8a0f43da81cd08645b7a710099ffe9ad1126b @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P a8a042a751557d06bf04455ed7629cb29adcd87f 549f4fd00d8325c10099b100e5202b77ee1d83ad -R f5be37b531ffc02ec5dfb73b01bae4f2 +P 9c572d424a20b0585bfac358a5d1ee5276dd05ba +R 729d7ae625d9d3f1fd7b4a057f6fc25f U drh -Z a17a96e51ae999984897d31a6e51ca34 +Z 5a28382a5967a16e17f8380cc4335d77 diff --git a/manifest.uuid b/manifest.uuid index 28f74b0a56..3f5736052d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c572d424a20b0585bfac358a5d1ee5276dd05ba \ No newline at end of file +a82938731b21d6166d7d482994cb065c8b725083 \ No newline at end of file diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index cf59257175..6859f6c517 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -93,6 +93,31 @@ ** ** This last query will show up to 50 words out of the vocabulary that ** match or nearly match the $prefix. +** +** MULTIPLE RULE SETS +** +** An enhancement as of 2012-02-14 allows multiple rule sets to coexist in +** the same fuzzer. This allows, for example, the fuzzer to operate in +** multiple languages. +** +** A new column "ruleset" is added to the table. This column must have a +** value between 0 and 49. The default value for the ruleset is 0. But +** alternative values can be specified. For example: +** +** INSERT INTO f(ruleset,cFrom,cTo,Cost) VALUES(1,'qu','k',100); +** +** Only one ruleset will be used at a time. When running a MATCH query, +** specify the desired ruleset using a "ruleset=N" term in the WHERE clause. +** For example: +** +** SELECT vocabulary.w FROM f, vocabulary +** WHERE f.word MATCH $word +** AND f.distance<=200 +** AND f.word=vocabulary.w +** AND f.ruleset=1 -- Specify the ruleset to use here +** LIMIT 20 +** +** If no ruleset is specified in the WHERE clause, ruleset 0 is used. */ #include "sqlite3.h" #include @@ -112,10 +137,24 @@ typedef struct fuzzer_seen fuzzer_seen; typedef struct fuzzer_stem fuzzer_stem; /* -** Type of the "cost" of an edit operation. Might be changed to -** "float" or "double" or "sqlite3_int64" in the future. +** Various types. +** +** fuzzer_cost is the "cost" of an edit operation. +** +** fuzzer_len is the length of a matching string. +** +** fuzzer_ruleid is an ruleset identifier. */ typedef int fuzzer_cost; +typedef signed char fuzzer_len; +typedef unsigned char fuzzer_ruleid; + +/* +** Limits +*/ +#define FUZZER_MX_LENGTH 50 /* Maximum length of a search string */ +#define FUZZER_MX_RULEID 50 /* Maximum rule ID */ +#define FUZZER_MX_COST 1000 /* Maximum single-rule cost */ /* @@ -123,11 +162,12 @@ typedef int fuzzer_cost; ** All rules are kept on a linked list sorted by rCost. */ struct fuzzer_rule { - fuzzer_rule *pNext; /* Next rule in order of increasing rCost */ - fuzzer_cost rCost; /* Cost of this transformation */ - int nFrom, nTo; /* Length of the zFrom and zTo strings */ - char *zFrom; /* Transform from */ - char zTo[4]; /* Transform to (extra space appended) */ + fuzzer_rule *pNext; /* Next rule in order of increasing rCost */ + char *zFrom; /* Transform from */ + fuzzer_cost rCost; /* Cost of this transformation */ + fuzzer_len nFrom, nTo; /* Length of the zFrom and zTo strings */ + fuzzer_ruleid iRuleset; /* The rule set to which this rule belongs */ + char zTo[4]; /* Transform to (extra space appended) */ }; /* @@ -143,13 +183,13 @@ struct fuzzer_rule { */ struct fuzzer_stem { char *zBasis; /* Word being fuzzed */ - int nBasis; /* Length of the zBasis string */ const fuzzer_rule *pRule; /* Current rule to apply */ - int n; /* Apply pRule at this character offset */ - fuzzer_cost rBaseCost; /* Base cost of getting to zBasis */ - fuzzer_cost rCostX; /* Precomputed rBaseCost + pRule->rCost */ fuzzer_stem *pNext; /* Next stem in rCost order */ fuzzer_stem *pHash; /* Next stem with same hash on zBasis */ + fuzzer_cost rBaseCost; /* Base cost of getting to zBasis */ + fuzzer_cost rCostX; /* Precomputed rBaseCost + pRule->rCost */ + fuzzer_len nBasis; /* Length of the zBasis string */ + fuzzer_len n; /* Apply pRule at this character offset */ }; /* @@ -179,6 +219,7 @@ struct fuzzer_cursor { char *zBuf; /* Temporary use buffer */ int nBuf; /* Bytes allocated for zBuf */ int nStem; /* Number of stems allocated */ + int iRuleset; /* Only process rules from this ruleset */ fuzzer_rule nullRule; /* Null rule used first */ fuzzer_stem *apHash[FUZZER_HASH]; /* Hash of previously generated terms */ }; @@ -202,7 +243,8 @@ static int fuzzerConnect( if( pNew==0 ) return SQLITE_NOMEM; pNew->zClassName = (char*)&pNew[1]; memcpy(pNew->zClassName, argv[0], n); - sqlite3_declare_vtab(db, "CREATE TABLE x(word,distance,cFrom,cTo,cost)"); + sqlite3_declare_vtab(db, + "CREATE TABLE x(word,distance,ruleset,cFrom,cTo,cost)"); memset(pNew, 0, sizeof(*pNew)); *ppVtab = &pNew->base; return SQLITE_OK; @@ -424,7 +466,7 @@ static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ } h = fuzzerHash(pCur->zBuf); pLookup = pCur->apHash[h]; - while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ + while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ pLookup = pLookup->pHash; } return pLookup!=0; @@ -453,8 +495,11 @@ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ } } pStem->n = -1; - pStem->pRule = pRule->pNext; - if( pStem->pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; + do{ + pRule = pRule->pNext; + }while( pRule && pRule->iRuleset!=pCur->iRuleset ); + pStem->pRule = pRule; + if( pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; } return 0; } @@ -667,16 +712,22 @@ static int fuzzerFilter( fuzzer_cursor *pCur = (fuzzer_cursor *)pVtabCursor; const char *zWord = 0; fuzzer_stem *pStem; + int idx; fuzzerClearCursor(pCur, 1); pCur->rLimit = 2147483647; - if( idxNum==1 ){ + idx = 0; + if( idxNum & 1 ){ zWord = (const char*)sqlite3_value_text(argv[0]); - }else if( idxNum==2 ){ - pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[0]); - }else if( idxNum==3 ){ - zWord = (const char*)sqlite3_value_text(argv[0]); - pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[1]); + idx++; + } + if( idxNum & 2 ){ + pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[idx]); + idx++; + } + if( idxNum & 4 ){ + pCur->iRuleset = (fuzzer_cost)sqlite3_value_int(argv[idx]); + idx++; } if( zWord==0 ) zWord = ""; pCur->pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); @@ -735,22 +786,29 @@ static int fuzzerEof(sqlite3_vtab_cursor *cur){ /* ** Search for terms of these forms: ** -** word MATCH $str -** distance < $value -** distance <= $value +** (A) word MATCH $str +** (B1) distance < $value +** (B2) distance <= $value +** (C) ruleid == $ruleid ** ** The distance< and distance<= are both treated as distance<=. -** The query plan number is as follows: +** The query plan number is a bit vector: ** -** 0: None of the terms above are found -** 1: There is a "word MATCH" term with $str in filter.argv[0]. -** 2: There is a "distance<" term with $value in filter.argv[0]. -** 3: Both "word MATCH" and "distance<" with $str in argv[0] and -** $value in argv[1]. +** bit 1: Term of the form (A) found +** bit 2: Term like (B1) or (B2) found +** bit 3: Term like (C) found +** +** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set +** then $value is in filter.argv[0] if bit-1 is clear and is in +** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is +** in filter.argv[0] if bit-1 and bit-2 are both zero, is in +** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in +** filter.argv[2] if both bit-1 and bit-2 are set. */ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iPlan = 0; int iDistTerm = -1; + int iRulesetTerm = -1; int i; const struct sqlite3_index_constraint *pConstraint; pConstraint = pIdxInfo->aConstraint; @@ -772,11 +830,23 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ iPlan |= 2; iDistTerm = i; } + if( (iPlan & 4)==0 + && pConstraint->iColumn==2 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + pIdxInfo->aConstraintUsage[i].omit = 1; + iRulesetTerm = i; + } } - if( iPlan==2 ){ - pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1; - }else if( iPlan==3 ){ - pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 2; + if( iPlan & 2 ){ + pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0); + } + if( iPlan & 4 ){ + int idx = 1; + if( iPlan & 1 ) idx++; + if( iPlan & 2 ) idx++; + pIdxInfo->aConstraintUsage[iRulesetTerm].argvIndex = idx; } pIdxInfo->idxNum = iPlan; if( pIdxInfo->nOrderBy==1 @@ -811,7 +881,8 @@ static int fuzzerUpdate( const char *zTo; int nTo; fuzzer_cost rCost; - if( argc!=7 ){ + int rulesetId; + if( argc!=8 ){ sqlite3_free(pVTab->zErrMsg); pVTab->zErrMsg = sqlite3_mprintf("cannot delete from a %s virtual table", p->zClassName); @@ -823,22 +894,36 @@ static int fuzzerUpdate( p->zClassName); return SQLITE_CONSTRAINT; } - zFrom = (char*)sqlite3_value_text(argv[4]); + zFrom = (char*)sqlite3_value_text(argv[5]); if( zFrom==0 ) zFrom = ""; - zTo = (char*)sqlite3_value_text(argv[5]); + zTo = (char*)sqlite3_value_text(argv[6]); if( zTo==0 ) zTo = ""; if( strcmp(zFrom,zTo)==0 ){ /* Silently ignore null transformations */ return SQLITE_OK; } - rCost = sqlite3_value_int(argv[6]); - if( rCost<=0 ){ + rCost = sqlite3_value_int(argv[7]); + if( rCost<=0 || rCost>FUZZER_MX_COST ){ sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("cost must be positive"); + pVTab->zErrMsg = sqlite3_mprintf("cost must be between 1 and %d", + FUZZER_MX_COST); return SQLITE_CONSTRAINT; } nFrom = strlen(zFrom); nTo = strlen(zTo); + if( nFrom>FUZZER_MX_LENGTH || nTo>FUZZER_MX_LENGTH ){ + sqlite3_free(pVTab->zErrMsg); + pVTab->zErrMsg = sqlite3_mprintf("maximum string length is %d", + FUZZER_MX_LENGTH); + return SQLITE_CONSTRAINT; + } + rulesetId = sqlite3_value_int(argv[4]); + if( rulesetId<0 || rulesetId>FUZZER_MX_RULEID ){ + sqlite3_free(pVTab->zErrMsg); + pVTab->zErrMsg = sqlite3_mprintf("rulesetid must be between 0 and %d", + FUZZER_MX_RULEID); + return SQLITE_CONSTRAINT; + } pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); if( pRule==0 ){ return SQLITE_NOMEM; @@ -850,6 +935,7 @@ static int fuzzerUpdate( pRule->nTo = nTo; pRule->rCost = rCost; pRule->pNext = p->pNewRule; + pRule->iRuleset = rulesetId; p->pNewRule = pRule; return SQLITE_OK; } diff --git a/test/fuzzer1.test b/test/fuzzer1.test index 6c23211859..d3199697e8 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -43,6 +43,64 @@ do_test fuzzer1-1.3 { } } {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100 ebcdo 110 obcde 110 obcda 111 obcdo 210} +do_test fuzzer1-1.4 { + db eval { + INSERT INTO f1(ruleset, cfrom, cto, cost) VALUES(1,'b','x',1); + INSERT INTO f1(ruleset, cfrom, cto, cost) VALUES(1,'d','y',10); + INSERT INTO f1(ruleset, cfrom, cto, cost) VALUES(1,'y','z',100); + } +} {} +do_test fuzzer1-1.5 { + db eval { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' + } +} {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100 ebcdo 110 obcde 110 obcda 111 obcdo 210} +do_test fuzzer1-1.6 { + db eval { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND ruleset=0 + } +} {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100 ebcdo 110 obcde 110 obcda 111 obcdo 210} +do_test fuzzer1-1.7 { + db eval { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND ruleset=1 + } +} {abcde 0 axcde 1 axcda 2 abcye 10 abcya 11 axcye 11 axcya 12 abcze 110 abcza 111 axcze 111 axcza 112} +do_test fuzzer1-1.8 { + db eval { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND distance<100 + } +} {abcde 0 abcda 1 ebcde 10 ebcda 11} +do_test fuzzer1-1.9 { + db eval { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND distance<=100 + } +} {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100} +do_test fuzzer1-1.10 { + db eval { + SELECT word, distance FROM f1 + WHERE word MATCH 'abcde' AND distance<100 AND ruleset=0 + } +} {abcde 0 abcda 1 ebcde 10 ebcda 11} +do_test fuzzer1-1.11 { + db eval { + SELECT word, distance FROM f1 + WHERE word MATCH 'abcde' AND distance<=100 AND ruleset=0 + } +} {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100} +do_test fuzzer1-1.12 { + db eval { + SELECT word, distance FROM f1 + WHERE word MATCH 'abcde' AND distance<12 AND ruleset=1 + } +} {abcde 0 axcde 1 axcda 2 abcye 10 abcya 11 axcye 11} +do_test fuzzer1-1.13 { + db eval { + SELECT word, distance FROM f1 + WHERE word MATCH 'abcde' AND distance<=12 AND ruleset=1 + } +} {abcde 0 axcde 1 axcda 2 abcye 10 abcya 11 axcye 11 axcya 12} + + do_test fuzzer1-2.0 { execsql { CREATE VIRTUAL TABLE temp.f2 USING fuzzer; From 6e5c5f4124656c86ef36e2887030e0ec1ccb65eb Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Feb 2012 18:56:01 +0000 Subject: [PATCH 16/51] Increase the maximum ruleset id in the fuzzer from 50 to 2^31-1. FossilOrigin-Name: 760e009adc6d0fffb8e6f64c7ec283938a417a77 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_fuzzer.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d92a333bed..70772b6d29 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sfuzzer\svirtual\stable\sto\ssupport\smultiple\srule\ssets. -D 2012-02-14T15:34:50.192 +C Increase\sthe\smaximum\sruleset\sid\sin\sthe\sfuzzer\sfrom\s50\sto\s2^31-1. +D 2012-02-14T18:56:01.799 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/test_config.c a036a69b550ebc477ab9ca2b37269201f888436e F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 6232d722a4ddb193035aa13a03796bf57d6c12fd -F src/test_fuzzer.c 0b11b466bd9c5dc8d882d29bb8c7e576963fd905 +F src/test_fuzzer.c 2f5d4cf96e54c96c88560cd04ba533e8749efb4b F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 9c572d424a20b0585bfac358a5d1ee5276dd05ba -R 729d7ae625d9d3f1fd7b4a057f6fc25f +P a82938731b21d6166d7d482994cb065c8b725083 +R 72ac1f4c7362fe7e245ef7893bc4161c U drh -Z 5a28382a5967a16e17f8380cc4335d77 +Z f9c57e19d8929af663421df88756961b diff --git a/manifest.uuid b/manifest.uuid index 3f5736052d..76a7bdb966 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a82938731b21d6166d7d482994cb065c8b725083 \ No newline at end of file +760e009adc6d0fffb8e6f64c7ec283938a417a77 \ No newline at end of file diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index 6859f6c517..892249b956 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -147,14 +147,14 @@ typedef struct fuzzer_stem fuzzer_stem; */ typedef int fuzzer_cost; typedef signed char fuzzer_len; -typedef unsigned char fuzzer_ruleid; +typedef int fuzzer_ruleid; /* ** Limits */ -#define FUZZER_MX_LENGTH 50 /* Maximum length of a search string */ -#define FUZZER_MX_RULEID 50 /* Maximum rule ID */ -#define FUZZER_MX_COST 1000 /* Maximum single-rule cost */ +#define FUZZER_MX_LENGTH 50 /* Maximum length of a search string */ +#define FUZZER_MX_RULEID 2147483647 /* Maximum rule ID */ +#define FUZZER_MX_COST 1000 /* Maximum single-rule cost */ /* From 75f8f75f1e80dc9ab75cbec9aa6a1504fd9c10f6 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Feb 2012 19:36:09 +0000 Subject: [PATCH 17/51] Fix a case in test_fuzzer.c causing transformations from the wrong ruleset to be applied in some cases. FossilOrigin-Name: cb5f5ebc563b8d3e47bc30b6dbb374bb91efd3ef --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/test_fuzzer.c | 4 ++++ test/fuzzer1.test | 19 +++++++++++++------ 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 70772b6d29..a05f010578 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\smaximum\sruleset\sid\sin\sthe\sfuzzer\sfrom\s50\sto\s2^31-1. -D 2012-02-14T18:56:01.799 +C Fix\sa\scase\sin\stest_fuzzer.c\scausing\stransformations\sfrom\sthe\swrong\sruleset\sto\sbe\sapplied\sin\ssome\scases. +D 2012-02-20T19:36:09.428 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/test_config.c a036a69b550ebc477ab9ca2b37269201f888436e F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 6232d722a4ddb193035aa13a03796bf57d6c12fd -F src/test_fuzzer.c 2f5d4cf96e54c96c88560cd04ba533e8749efb4b +F src/test_fuzzer.c 5c34fdb55c4fa3090d076886139b1f633327a2c5 F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -504,7 +504,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzer1.test ff725a0eec070dfc2b2acc13b21a52a139382929 +F test/fuzzer1.test d5638894ffd89cb01e1276e2a52707b0b1261fe2 F test/hook.test 5f3749de6462a6b87b4209b74adf7df5ac2df639 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test a7b8a0f43da81cd08645b7a710099ffe9ad1126b @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P a82938731b21d6166d7d482994cb065c8b725083 -R 72ac1f4c7362fe7e245ef7893bc4161c -U drh -Z f9c57e19d8929af663421df88756961b +P 760e009adc6d0fffb8e6f64c7ec283938a417a77 +R 2f925ccdeb58b2c1514de71ba4c4df79 +U dan +Z 2c1d8a850f278c0cc6fb31de180d043b diff --git a/manifest.uuid b/manifest.uuid index 76a7bdb966..df025c2641 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -760e009adc6d0fffb8e6f64c7ec283938a417a77 \ No newline at end of file +cb5f5ebc563b8d3e47bc30b6dbb374bb91efd3ef \ No newline at end of file diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index 892249b956..4402a245ef 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -480,6 +480,7 @@ static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ const fuzzer_rule *pRule; while( (pRule = pStem->pRule)!=0 ){ + assert( pRule==&pCur->nullRule || pRule->iRuleset==pCur->iRuleset ); while( pStem->n < pStem->nBasis - pRule->nFrom ){ pStem->n++; if( pRule->nFrom==0 @@ -626,6 +627,9 @@ static fuzzer_stem *fuzzerNewStem( pNew->nBasis = strlen(zWord); memcpy(pNew->zBasis, zWord, pNew->nBasis+1); pNew->pRule = pCur->pVtab->pRule; + while( pNew->pRule && pNew->pRule->iRuleset!=pCur->iRuleset ){ + pNew->pRule = pNew->pRule->pNext; + } pNew->n = -1; pNew->rBaseCost = pNew->rCostX = rBaseCost; h = fuzzerHash(pNew->zBasis); diff --git a/test/fuzzer1.test b/test/fuzzer1.test index d3199697e8..05c00bc025 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -64,7 +64,7 @@ do_test fuzzer1-1.7 { db eval { SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND ruleset=1 } -} {abcde 0 axcde 1 axcda 2 abcye 10 abcya 11 axcye 11 axcya 12 abcze 110 abcza 111 axcze 111 axcza 112} +} {abcde 0 axcde 1 abcye 10 axcye 11 abcze 110 axcze 111} do_test fuzzer1-1.8 { db eval { SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND distance<100 @@ -90,16 +90,15 @@ do_test fuzzer1-1.11 { do_test fuzzer1-1.12 { db eval { SELECT word, distance FROM f1 - WHERE word MATCH 'abcde' AND distance<12 AND ruleset=1 + WHERE word MATCH 'abcde' AND distance<11 AND ruleset=1 } -} {abcde 0 axcde 1 axcda 2 abcye 10 abcya 11 axcye 11} +} {abcde 0 axcde 1 abcye 10} do_test fuzzer1-1.13 { db eval { SELECT word, distance FROM f1 - WHERE word MATCH 'abcde' AND distance<=12 AND ruleset=1 + WHERE word MATCH 'abcde' AND distance<=11 AND ruleset=1 } -} {abcde 0 axcde 1 axcda 2 abcye 10 abcya 11 axcye 11 axcya 12} - +} {abcde 0 axcde 1 abcye 10 axcye 11} do_test fuzzer1-2.0 { execsql { @@ -1437,4 +1436,12 @@ do_test fuzzer1-2.3 { } {{tyler finley} trailer taymouth steelewood tallia tallu talwyn thelema} +do_execsql_test fuzzer1-3.1 { + CREATE VIRTUAL TABLE temp.f3 USING fuzzer; + CREATE TABLE f3(ruleset, cfrom, cto, cost); + INSERT INTO f3(ruleset, cfrom, cto, cost) VALUES(0, 'x','y', 10); + INSERT INTO f3(ruleset, cfrom, cto, cost) VALUES(1, 'a','b', 10); + SELECT word FROM f3 WHERE word MATCH 'ax' +} {ax ay} + finish_test From a8ab692fa2e0b76f3ab713aad7b3c9e7c5537838 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Feb 2012 20:03:48 +0000 Subject: [PATCH 18/51] Change the way the fuzzer (test_fuzzer.c) works so that it loads its configuration from a database table. FossilOrigin-Name: 90b7b957f8933047fd2878048dfa3ec4891988b8 --- manifest | 14 +- manifest.uuid | 2 +- src/test_fuzzer.c | 449 +++++++++++++++++++++++++++------------------- test/fuzzer1.test | 272 ++++++++++++++++------------ 4 files changed, 433 insertions(+), 304 deletions(-) diff --git a/manifest b/manifest index a05f010578..b0a676f806 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\sin\stest_fuzzer.c\scausing\stransformations\sfrom\sthe\swrong\sruleset\sto\sbe\sapplied\sin\ssome\scases. -D 2012-02-20T19:36:09.428 +C Change\sthe\sway\sthe\sfuzzer\s(test_fuzzer.c)\sworks\sso\sthat\sit\sloads\sits\sconfiguration\sfrom\sa\sdatabase\stable. +D 2012-02-20T20:03:48.835 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/test_config.c a036a69b550ebc477ab9ca2b37269201f888436e F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 6232d722a4ddb193035aa13a03796bf57d6c12fd -F src/test_fuzzer.c 5c34fdb55c4fa3090d076886139b1f633327a2c5 +F src/test_fuzzer.c 010ee3d4122fd955d6f0db598f68d62f95d15fa9 F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -504,7 +504,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzer1.test d5638894ffd89cb01e1276e2a52707b0b1261fe2 +F test/fuzzer1.test 50a480932b91df9d61dd089f338e448991ab771e F test/hook.test 5f3749de6462a6b87b4209b74adf7df5ac2df639 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test a7b8a0f43da81cd08645b7a710099ffe9ad1126b @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 760e009adc6d0fffb8e6f64c7ec283938a417a77 -R 2f925ccdeb58b2c1514de71ba4c4df79 +P cb5f5ebc563b8d3e47bc30b6dbb374bb91efd3ef +R 15ed6cfb8d0ad7f6ea31fbc888768f7f U dan -Z 2c1d8a850f278c0cc6fb31de180d043b +Z d7a1a4dc789683dc31afc336a51e56bf diff --git a/manifest.uuid b/manifest.uuid index df025c2641..8798c458c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb5f5ebc563b8d3e47bc30b6dbb374bb91efd3ef \ No newline at end of file +90b7b957f8933047fd2878048dfa3ec4891988b8 \ No newline at end of file diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index 4402a245ef..6953965243 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -10,43 +10,56 @@ ** ************************************************************************* ** -** Code for demonstartion virtual table that generates variations +** Code for a demonstration virtual table that generates variations ** on an input word at increasing edit distances from the original. ** ** A fuzzer virtual table is created like this: ** -** CREATE VIRTUAL TABLE temp.f USING fuzzer; +** CREATE VIRTUAL TABLE f USING fuzzer(); ** -** The name of the new virtual table in the example above is "f". -** Note that all fuzzer virtual tables must be TEMP tables. The -** "temp." prefix in front of the table name is required when the -** table is being created. The "temp." prefix can be omitted when -** using the table as long as the name is unambiguous. +** When it is created, the new fuzzer table must be supplied with the +** name of a "fuzzer data table", which must reside in the same database +** file as the new fuzzer table. The fuzzer data table contains the various +** transformations and their costs that the fuzzer logic uses to generate +** variations. ** -** Before being used, the fuzzer needs to be programmed by giving it -** character transformations and a cost associated with each transformation. -** Examples: +** The fuzzer data table must contain exactly four columns (more precisely, +** the statement "SELECT * FROM " must return records +** that consist of four columns). It does not matter what the columns are +** named. ** -** INSERT INTO f(cFrom,cTo,Cost) VALUES('','a',100); +** Each row in the fuzzer table represents a single character transformation. +** The left most column of the row (column 0) contains an integer value - +** the identifier of the ruleset to which the transformation rule belongs +** (see "MULTIPLE RULE SETS" below). The second column of the row (column 0) +** contains the input character or characters. The third column contains the +** output character or characters. And the fourth column contains the integer +** cost of making the transformation. For example: ** -** The above statement says that the cost of inserting a letter 'a' is -** 100. (All costs are integers. We recommend that costs be scaled so -** that the average cost is around 100.) +** CREATE TABLE f_data(ruleset, cFrom, cTo, Cost); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'b', '', 87); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40); ** -** INSERT INTO f(cFrom,cTo,Cost) VALUES('b','',87); -** -** The above statement says that the cost of deleting a single letter -** 'b' is 87. -** -** INSERT INTO f(cFrom,cTo,Cost) VALUES('o','oe',38); -** INSERT INTO f(cFrom,cTo,Cost) VALUES('oe','o',40); -** -** This third example says that the cost of transforming the single -** letter "o" into the two-letter sequence "oe" is 38 and that the +** The first row inserted into the fuzzer data table by the SQL script +** above indicates that the cost of inserting a letter 'a' is 100. (All +** costs are integers. We recommend that costs be scaled so that the +** average cost is around 100.) The second INSERT statement creates a rule +** that the cost of that the cost of deleting a single letter 'b' is 87. +** The third and fourth INSERT statements mean that the cost of transforming +** a single letter "o" into the two-letter sequence "oe" is 38 and that the ** cost of transforming "oe" back into "o" is 40. ** -** After all the transformation costs have been set, the fuzzer table -** can be queried as follows: +** The contents of the fuzzer data table are loaded into main memory when +** a fuzzer table is first created, and may be internally reloaded by the +** system at any subsequent time. Therefore, the fuzzer data table should be +** populated before the fuzzer table is created and not modified thereafter. +** If you do need to modify the contents of the fuzzer data table, it is +** recommended that the associated fuzzer table be dropped, the fuzzer data +** table edited, and the fuzzer table recreated within a single transaction. +** +** Once it has been created, the fuzzer table can be queried as follows: ** ** SELECT word, distance FROM f ** WHERE word MATCH 'abcdefg' @@ -96,19 +109,16 @@ ** ** MULTIPLE RULE SETS ** -** An enhancement as of 2012-02-14 allows multiple rule sets to coexist in -** the same fuzzer. This allows, for example, the fuzzer to operate in +** Normally, the "ruleset" value associated with all character transformations +** in the fuzzer data table is zero. However, if required, the fuzzer table +** allows multiple rulesets to be defined. Each query uses only a single +** ruleset. This allows, for example, a single fuzzer table to support ** multiple languages. ** -** A new column "ruleset" is added to the table. This column must have a -** value between 0 and 49. The default value for the ruleset is 0. But -** alternative values can be specified. For example: -** -** INSERT INTO f(ruleset,cFrom,cTo,Cost) VALUES(1,'qu','k',100); -** -** Only one ruleset will be used at a time. When running a MATCH query, -** specify the desired ruleset using a "ruleset=N" term in the WHERE clause. -** For example: +** By default, only the rules from ruleset 0 are used. To specify an +** alternative ruleset, a "ruleset = ?" expression must be added to the +** WHERE clause of a SELECT, where ? is the identifier of the desired +** ruleset. For example: ** ** SELECT vocabulary.w FROM f, vocabulary ** WHERE f.word MATCH $word @@ -117,7 +127,8 @@ ** AND f.ruleset=1 -- Specify the ruleset to use here ** LIMIT 20 ** -** If no ruleset is specified in the WHERE clause, ruleset 0 is used. +** If no "ruleset = ?" constraint is specified in the WHERE clause, ruleset +** 0 is used. */ #include "sqlite3.h" #include @@ -199,7 +210,6 @@ struct fuzzer_vtab { sqlite3_vtab base; /* Base class - must be first */ char *zClassName; /* Name of this class. Default: "fuzzer" */ fuzzer_rule *pRule; /* All active rules in this fuzzer */ - fuzzer_rule *pNewRule; /* New rules to add when last cursor expires */ int nCursor; /* Number of active cursors */ }; @@ -224,51 +234,6 @@ struct fuzzer_cursor { fuzzer_stem *apHash[FUZZER_HASH]; /* Hash of previously generated terms */ }; -/* Methods for the fuzzer module */ -static int fuzzerConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - fuzzer_vtab *pNew; - int n; - if( strcmp(argv[1],"temp")!=0 ){ - *pzErr = sqlite3_mprintf("%s virtual tables must be TEMP", argv[0]); - return SQLITE_ERROR; - } - n = strlen(argv[0]) + 1; - pNew = sqlite3_malloc( sizeof(*pNew) + n ); - if( pNew==0 ) return SQLITE_NOMEM; - pNew->zClassName = (char*)&pNew[1]; - memcpy(pNew->zClassName, argv[0], n); - sqlite3_declare_vtab(db, - "CREATE TABLE x(word,distance,ruleset,cFrom,cTo,cost)"); - memset(pNew, 0, sizeof(*pNew)); - *ppVtab = &pNew->base; - return SQLITE_OK; -} -/* Note that for this virtual table, the xCreate and xConnect -** methods are identical. */ - -static int fuzzerDisconnect(sqlite3_vtab *pVtab){ - fuzzer_vtab *p = (fuzzer_vtab*)pVtab; - assert( p->nCursor==0 ); - do{ - while( p->pRule ){ - fuzzer_rule *pRule = p->pRule; - p->pRule = pRule->pNext; - sqlite3_free(pRule); - } - p->pRule = p->pNewRule; - p->pNewRule = 0; - }while( p->pRule ); - sqlite3_free(p); - return SQLITE_OK; -} -/* The xDisconnect and xDestroy methods are also the same */ - /* ** The two input rule lists are both sorted in order of increasing ** cost. Merge them together into a single list, sorted by cost, and @@ -298,6 +263,218 @@ static fuzzer_rule *fuzzerMergeRules(fuzzer_rule *pA, fuzzer_rule *pB){ return head.pNext; } +/* +** Statement pStmt currently points to a row in the fuzzer data table. This +** function allocates and populates a fuzzer_rule structure according to +** the content of the row. +** +** If successful, *ppRule is set to point to the new object and SQLITE_OK +** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point +** to an error message and an SQLite error code returned. +*/ +static int fuzzerLoadOneRule( + fuzzer_vtab *p, /* Fuzzer virtual table handle */ + sqlite3_stmt *pStmt, /* Base rule on statements current row */ + fuzzer_rule **ppRule, /* OUT: New rule object */ + char **pzErr /* OUT: Error message */ +){ + int iRuleset = sqlite3_column_int(pStmt, 0); + const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); + const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); + int nCost = sqlite3_column_int(pStmt, 3); + + int rc = SQLITE_OK; /* Return code */ + int nFrom; /* Size of string zFrom, in bytes */ + int nTo; /* Size of string zTo, in bytes */ + fuzzer_rule *pRule = 0; /* New rule object to return */ + + if( zFrom==0 ) zFrom = ""; + if( zTo==0 ) zTo = ""; + nFrom = strlen(zFrom); + nTo = strlen(zTo); + + /* Silently ignore null transformations */ + if( strcmp(zFrom, zTo)==0 ){ + *ppRule = 0; + return SQLITE_OK; + } + + if( nCost<=0 || nCost>FUZZER_MX_COST ){ + *pzErr = sqlite3_mprintf("cost must be between 1 and %d", FUZZER_MX_COST); + rc = SQLITE_ERROR; + }else + if( nFrom>FUZZER_MX_LENGTH || nTo>FUZZER_MX_LENGTH ){ + *pzErr = sqlite3_mprintf("maximum string length is %d", FUZZER_MX_LENGTH); + rc = SQLITE_ERROR; + }else + if( iRuleset<0 || iRuleset>FUZZER_MX_RULEID ){ + *pzErr = sqlite3_mprintf( + "ruleset must be between 0 and %d", FUZZER_MX_RULEID); + rc = SQLITE_ERROR; + }else{ + + pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); + if( pRule==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRule, 0, sizeof(*pRule)); + pRule->zFrom = &pRule->zTo[nTo+1]; + pRule->nFrom = nFrom; + memcpy(pRule->zFrom, zFrom, nFrom+1); + memcpy(pRule->zTo, zTo, nTo+1); + pRule->nTo = nTo; + pRule->rCost = nCost; + pRule->iRuleset = iRuleset; + } + } + + *ppRule = pRule; + return rc; +} + +/* +** Load the content of the fuzzer data table into memory. +*/ +static int fuzzerLoadRules( + sqlite3 *db, /* Database handle */ + fuzzer_vtab *p, /* Virtual fuzzer table to configure */ + const char *zDb, /* Database containing rules data */ + const char *zData, /* Table containing rules data */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* SELECT used to read from rules table */ + fuzzer_rule *pHead = 0; + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zData); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int rc2; /* finalize() return code */ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db)); + }else if( sqlite3_column_count(pStmt)!=4 ){ + *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4", + p->zClassName, zData, sqlite3_column_count(pStmt) + ); + rc = SQLITE_ERROR; + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + fuzzer_rule *pRule = 0; + rc = fuzzerLoadOneRule(p, pStmt, &pRule, pzErr); + if( pRule ){ + pRule->pNext = pHead; + pHead = pRule; + } + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + sqlite3_free(zSql); + + /* All rules are now in a singly linked list starting at pHead. This + ** block sorts them by cost and then sets fuzzer_vtab.pRule to point to + ** point to the head of the sorted list. + */ + if( rc==SQLITE_OK ){ + unsigned int i; + fuzzer_rule *pX; + fuzzer_rule *a[15]; + for(i=0; ipNext; + pX->pNext = 0; + for(i=0; a[i] && ipRule = fuzzerMergeRules(p->pRule, pX); + }else{ + /* An error has occurred. Setting p->pRule to point to the head of the + ** allocated list ensures that the list will be cleaned up in this case. + */ + assert( p->pRule==0 ); + p->pRule = pHead; + } + + return rc; +} + + +/* +** xConnect/xCreate method for the fuzzer module. Arguments are: +** +** argv[0] -> module name ("fuzzer") +** argv[1] -> database name +** argv[2] -> table name +** argv[3] -> fuzzer rule table name +*/ +static int fuzzerConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + fuzzer_vtab *pNew = 0; /* New virtual table */ + const char *zModule = argv[0]; + const char *zDb = argv[1]; + + if( argc!=4 ){ + *pzErr = sqlite3_mprintf( + "%s: wrong number of CREATE VIRTUAL TABLE arguments", zModule + ); + rc = SQLITE_ERROR; + }else{ + int nModule; /* Length of zModule, in bytes */ + + nModule = strlen(zModule); + pNew = sqlite3_malloc( sizeof(*pNew) + nModule + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(*pNew)); + pNew->zClassName = (char*)&pNew[1]; + memcpy(pNew->zClassName, zModule, nModule+1); + + rc = fuzzerLoadRules(db, pNew, zDb, argv[3], pzErr); + if( rc==SQLITE_OK ){ + sqlite3_declare_vtab(db, "CREATE TABLE x(word, distance,ruleset)"); + }else{ + sqlite3_free(pNew); + pNew = 0; + } + } + } + + *ppVtab = (sqlite3_vtab *)pNew; + return rc; +} +/* Note that for this virtual table, the xCreate and xConnect +** methods are identical. */ + +static int fuzzerDisconnect(sqlite3_vtab *pVtab){ + fuzzer_vtab *p = (fuzzer_vtab*)pVtab; + assert( p->nCursor==0 ); + while( p->pRule ){ + fuzzer_rule *pRule = p->pRule; + p->pRule = pRule->pNext; + sqlite3_free(pRule); + } + sqlite3_free(p); + return SQLITE_OK; +} +/* The xDisconnect and xDestroy methods are also the same */ + /* ** Open a new fuzzer cursor. @@ -310,25 +487,6 @@ static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ memset(pCur, 0, sizeof(*pCur)); pCur->pVtab = p; *ppCursor = &pCur->base; - if( p->nCursor==0 && p->pNewRule ){ - unsigned int i; - fuzzer_rule *pX; - fuzzer_rule *a[15]; - for(i=0; ipNewRule)!=0 ){ - p->pNewRule = pX->pNext; - pX->pNext = 0; - for(i=0; a[i] && ipRule = fuzzerMergeRules(p->pRule, pX); - } p->nCursor++; return SQLITE_OK; } @@ -479,6 +637,7 @@ static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ */ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ const fuzzer_rule *pRule; + const int iSet = pCur->iRuleset; while( (pRule = pStem->pRule)!=0 ){ assert( pRule==&pCur->nullRule || pRule->iRuleset==pCur->iRuleset ); while( pStem->n < pStem->nBasis - pRule->nFrom ){ @@ -498,7 +657,7 @@ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ pStem->n = -1; do{ pRule = pRule->pNext; - }while( pRule && pRule->iRuleset!=pCur->iRuleset ); + }while( pRule && pRule->iRuleset!=iSet ); pStem->pRule = pRule; if( pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; } @@ -864,86 +1023,6 @@ static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ return SQLITE_OK; } -/* -** Disallow all attempts to DELETE or UPDATE. Only INSERTs are allowed. -** -** On an insert, the cFrom, cTo, and cost columns are used to construct -** a new rule. All other columns are ignored. The rule is ignored -** if cFrom and cTo are identical. A NULL value for cFrom or cTo is -** interpreted as an empty string. The cost must be positive. -*/ -static int fuzzerUpdate( - sqlite3_vtab *pVTab, - int argc, - sqlite3_value **argv, - sqlite_int64 *pRowid -){ - fuzzer_vtab *p = (fuzzer_vtab*)pVTab; - fuzzer_rule *pRule; - const char *zFrom; - int nFrom; - const char *zTo; - int nTo; - fuzzer_cost rCost; - int rulesetId; - if( argc!=8 ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("cannot delete from a %s virtual table", - p->zClassName); - return SQLITE_CONSTRAINT; - } - if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("cannot update a %s virtual table", - p->zClassName); - return SQLITE_CONSTRAINT; - } - zFrom = (char*)sqlite3_value_text(argv[5]); - if( zFrom==0 ) zFrom = ""; - zTo = (char*)sqlite3_value_text(argv[6]); - if( zTo==0 ) zTo = ""; - if( strcmp(zFrom,zTo)==0 ){ - /* Silently ignore null transformations */ - return SQLITE_OK; - } - rCost = sqlite3_value_int(argv[7]); - if( rCost<=0 || rCost>FUZZER_MX_COST ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("cost must be between 1 and %d", - FUZZER_MX_COST); - return SQLITE_CONSTRAINT; - } - nFrom = strlen(zFrom); - nTo = strlen(zTo); - if( nFrom>FUZZER_MX_LENGTH || nTo>FUZZER_MX_LENGTH ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("maximum string length is %d", - FUZZER_MX_LENGTH); - return SQLITE_CONSTRAINT; - } - rulesetId = sqlite3_value_int(argv[4]); - if( rulesetId<0 || rulesetId>FUZZER_MX_RULEID ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf("rulesetid must be between 0 and %d", - FUZZER_MX_RULEID); - return SQLITE_CONSTRAINT; - } - pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); - if( pRule==0 ){ - return SQLITE_NOMEM; - } - pRule->zFrom = &pRule->zTo[nTo+1]; - pRule->nFrom = nFrom; - memcpy(pRule->zFrom, zFrom, nFrom+1); - memcpy(pRule->zTo, zTo, nTo+1); - pRule->nTo = nTo; - pRule->rCost = rCost; - pRule->pNext = p->pNewRule; - pRule->iRuleset = rulesetId; - p->pNewRule = pRule; - return SQLITE_OK; -} - /* ** A virtual table module that provides read-only access to a ** Tcl global variable namespace. @@ -962,7 +1041,7 @@ static sqlite3_module fuzzerModule = { fuzzerEof, /* xEof - check for end of scan */ fuzzerColumn, /* xColumn - read data */ fuzzerRowid, /* xRowid - read data */ - fuzzerUpdate, /* xUpdate - INSERT */ + 0, /* xUpdate */ 0, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ diff --git a/test/fuzzer1.test b/test/fuzzer1.test index 05c00bc025..cfa6b756d5 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -23,48 +23,96 @@ ifcapable !vtab { } register_fuzzer_module db -do_test fuzzer1-1.0 { - catchsql {CREATE VIRTUAL TABLE fault1 USING fuzzer;} -} {1 {fuzzer virtual tables must be TEMP}} -do_test fuzzer1-1.1 { - db eval {CREATE VIRTUAL TABLE temp.f1 USING fuzzer;} -} {} -do_test fuzzer1-1.2 { - db eval { - INSERT INTO f1(cfrom, cto, cost) VALUES('e','a',1); - INSERT INTO f1(cfrom, cto, cost) VALUES('a','e',10); - INSERT INTO f1(cfrom, cto, cost) VALUES('e','o',100); - } + + + +# Check configuration errors. +# +do_catchsql_test fuzzer1-1.1 { + CREATE VIRTUAL TABLE f USING fuzzer; +} {1 {fuzzer: wrong number of CREATE VIRTUAL TABLE arguments}} + +do_catchsql_test fuzzer1-1.2 { + CREATE VIRTUAL TABLE f USING fuzzer(one, two); +} {1 {fuzzer: wrong number of CREATE VIRTUAL TABLE arguments}} + +do_catchsql_test fuzzer1-1.3 { + CREATE VIRTUAL TABLE f USING fuzzer(nosuchtable); +} {1 {fuzzer: no such table: main.nosuchtable}} + +do_catchsql_test fuzzer1-1.4 { + CREATE TEMP TABLE nosuchtable(a, b, c, d); + CREATE VIRTUAL TABLE f USING fuzzer(nosuchtable); +} {1 {fuzzer: no such table: main.nosuchtable}} + +do_catchsql_test fuzzer1-1.5 { + DROP TABLE temp.nosuchtable; + CREATE TABLE nosuchtable(a, b, c, d); + CREATE VIRTUAL TABLE temp.f USING fuzzer(nosuchtable); +} {1 {fuzzer: no such table: temp.nosuchtable}} + +do_catchsql_test fuzzer1-1.6 { + DROP TABLE IF EXISTS f_rules; + CREATE TABLE f_rules(a, b, c); + CREATE VIRTUAL TABLE f USING fuzzer(f_rules); +} {1 {fuzzer: f_rules has 3 columns, expected 4}} + +do_catchsql_test fuzzer1-1.7 { + DROP TABLE IF EXISTS f_rules; + CREATE TABLE f_rules(a, b, c, d, e); + CREATE VIRTUAL TABLE f USING fuzzer(f_rules); +} {1 {fuzzer: f_rules has 5 columns, expected 4}} + + +do_execsql_test fuzzer1-2.1 { + CREATE TABLE f1_rules(ruleset DEFAULT 0, cfrom, cto, cost); + INSERT INTO f1_rules(cfrom, cto, cost) VALUES('e','a',1); + INSERT INTO f1_rules(cfrom, cto, cost) VALUES('a','e',10); + INSERT INTO f1_rules(cfrom, cto, cost) VALUES('e','o',100); + + CREATE VIRTUAL TABLE f1 USING fuzzer(f1_rules); } {} -do_test fuzzer1-1.3 { - db eval { +do_execsql_test fuzzer1-2.1 { SELECT word, distance FROM f1 WHERE word MATCH 'abcde' - } -} {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100 ebcdo 110 obcde 110 obcda 111 obcdo 210} +} { + abcde 0 abcda 1 ebcde 10 + ebcda 11 abcdo 100 ebcdo 110 + obcde 110 obcda 111 obcdo 210 +} -do_test fuzzer1-1.4 { - db eval { - INSERT INTO f1(ruleset, cfrom, cto, cost) VALUES(1,'b','x',1); - INSERT INTO f1(ruleset, cfrom, cto, cost) VALUES(1,'d','y',10); - INSERT INTO f1(ruleset, cfrom, cto, cost) VALUES(1,'y','z',100); - } +do_execsql_test fuzzer1-2.4 { + INSERT INTO f1_rules(ruleset, cfrom, cto, cost) VALUES(1,'b','x',1); + INSERT INTO f1_rules(ruleset, cfrom, cto, cost) VALUES(1,'d','y',10); + INSERT INTO f1_rules(ruleset, cfrom, cto, cost) VALUES(1,'y','z',100); + + DROP TABLE f1; + CREATE VIRTUAL TABLE f1 USING fuzzer(f1_rules); } {} -do_test fuzzer1-1.5 { - db eval { - SELECT word, distance FROM f1 WHERE word MATCH 'abcde' - } -} {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100 ebcdo 110 obcde 110 obcda 111 obcdo 210} -do_test fuzzer1-1.6 { - db eval { - SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND ruleset=0 - } -} {abcde 0 abcda 1 ebcde 10 ebcda 11 abcdo 100 ebcdo 110 obcde 110 obcda 111 obcdo 210} -do_test fuzzer1-1.7 { - db eval { - SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND ruleset=1 - } -} {abcde 0 axcde 1 abcye 10 axcye 11 abcze 110 axcze 111} + +do_execsql_test fuzzer1-2.5 { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' +} { + abcde 0 abcda 1 ebcde 10 + ebcda 11 abcdo 100 ebcdo 110 + obcde 110 obcda 111 obcdo 210 +} + +do_execsql_test fuzzer1-2.6 { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND ruleset=0 +} { + abcde 0 abcda 1 ebcde 10 + ebcda 11 abcdo 100 ebcdo 110 + obcde 110 obcda 111 obcdo 210 +} + +do_execsql_test fuzzer1-2.7 { + SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND ruleset=1 +} { + abcde 0 axcde 1 abcye 10 + axcye 11 abcze 110 axcze 111 +} + do_test fuzzer1-1.8 { db eval { SELECT word, distance FROM f1 WHERE word MATCH 'abcde' AND distance<100 @@ -102,77 +150,78 @@ do_test fuzzer1-1.13 { do_test fuzzer1-2.0 { execsql { - CREATE VIRTUAL TABLE temp.f2 USING fuzzer; -- costs based on English letter frequencies - INSERT INTO f2(cFrom,cTo,cost) VALUES('a','e',24); - INSERT INTO f2(cFrom,cTo,cost) VALUES('a','o',47); - INSERT INTO f2(cFrom,cTo,cost) VALUES('a','u',50); - INSERT INTO f2(cFrom,cTo,cost) VALUES('e','a',23); - INSERT INTO f2(cFrom,cTo,cost) VALUES('e','i',33); - INSERT INTO f2(cFrom,cTo,cost) VALUES('e','o',37); - INSERT INTO f2(cFrom,cTo,cost) VALUES('i','e',33); - INSERT INTO f2(cFrom,cTo,cost) VALUES('i','y',33); - INSERT INTO f2(cFrom,cTo,cost) VALUES('o','a',41); - INSERT INTO f2(cFrom,cTo,cost) VALUES('o','e',46); - INSERT INTO f2(cFrom,cTo,cost) VALUES('o','u',57); - INSERT INTO f2(cFrom,cTo,cost) VALUES('u','o',58); - INSERT INTO f2(cFrom,cTo,cost) VALUES('y','i',33); - - INSERT INTO f2(cFrom,cTo,cost) VALUES('t','th',70); - INSERT INTO f2(cFrom,cTo,cost) VALUES('th','t',66); + CREATE TEMP TABLE f2_rules(ruleset, cFrom, cTo, cost); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','e',24); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','o',47); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','u',50); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('e','a',23); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('e','i',33); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('e','o',37); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('i','e',33); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('i','y',33); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('o','a',41); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('o','e',46); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('o','u',57); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('u','o',58); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('y','i',33); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('t','th',70); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('th','t',66); - INSERT INTO f2(cFrom,cTo,cost) VALUES('a','',84); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','b',106); - INSERT INTO f2(cFrom,cTo,cost) VALUES('b','',106); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','c',94); - INSERT INTO f2(cFrom,cTo,cost) VALUES('c','',94); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','d',89); - INSERT INTO f2(cFrom,cTo,cost) VALUES('d','',89); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','e',83); - INSERT INTO f2(cFrom,cTo,cost) VALUES('e','',83); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','f',97); - INSERT INTO f2(cFrom,cTo,cost) VALUES('f','',97); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','g',99); - INSERT INTO f2(cFrom,cTo,cost) VALUES('g','',99); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','h',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('h','',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','i',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('i','',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','j',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('j','',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','k',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('k','',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','l',89); - INSERT INTO f2(cFrom,cTo,cost) VALUES('l','',89); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','m',96); - INSERT INTO f2(cFrom,cTo,cost) VALUES('m','',96); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','n',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('n','',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','o',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('o','',85); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','p',100); - INSERT INTO f2(cFrom,cTo,cost) VALUES('p','',100); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','q',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('q','',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','r',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('r','',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','s',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('s','',86); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','t',84); - INSERT INTO f2(cFrom,cTo,cost) VALUES('t','',84); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','u',94); - INSERT INTO f2(cFrom,cTo,cost) VALUES('u','',94); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','v',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('v','',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','w',96); - INSERT INTO f2(cFrom,cTo,cost) VALUES('w','',96); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','x',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('x','',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','y',100); - INSERT INTO f2(cFrom,cTo,cost) VALUES('y','',100); - INSERT INTO f2(cFrom,cTo,cost) VALUES('','z',120); - INSERT INTO f2(cFrom,cTo,cost) VALUES('z','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','',84); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','b',106); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('b','',106); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','c',94); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('c','',94); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','d',89); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('d','',89); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','e',83); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('e','',83); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','f',97); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('f','',97); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','g',99); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('g','',99); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','h',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('h','',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','i',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('i','',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','j',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('j','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','k',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('k','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','l',89); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('l','',89); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','m',96); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('m','',96); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','n',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('n','',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','o',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('o','',85); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','p',100); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('p','',100); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','q',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('q','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','r',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('r','',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','s',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('s','',86); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','t',84); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('t','',84); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','u',94); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('u','',94); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','v',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('v','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','w',96); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('w','',96); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','x',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('x','',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','y',100); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('y','',100); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','z',120); + INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('z','',120); + + CREATE VIRTUAL TABLE temp.f2 USING fuzzer(f2_rules); -- Street names for the 28269 ZIPCODE. -- @@ -1435,12 +1484,13 @@ do_test fuzzer1-2.3 { } } {{tyler finley} trailer taymouth steelewood tallia tallu talwyn thelema} - -do_execsql_test fuzzer1-3.1 { - CREATE VIRTUAL TABLE temp.f3 USING fuzzer; - CREATE TABLE f3(ruleset, cfrom, cto, cost); - INSERT INTO f3(ruleset, cfrom, cto, cost) VALUES(0, 'x','y', 10); - INSERT INTO f3(ruleset, cfrom, cto, cost) VALUES(1, 'a','b', 10); +forcedelete test.db2 +do_execsql_test fuzzer1-4.1 { + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.f3_rules(ruleset, cfrom, cto, cost); + INSERT INTO f3_rules(ruleset, cfrom, cto, cost) VALUES(0, 'x','y', 10); + INSERT INTO f3_rules(ruleset, cfrom, cto, cost) VALUES(1, 'a','b', 10); + CREATE VIRTUAL TABLE aux.f3 USING fuzzer(f3_rules); SELECT word FROM f3 WHERE word MATCH 'ax' } {ax ay} From 5577671da3e0c0d8ed2811443967e16333f30807 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Feb 2012 22:44:12 +0000 Subject: [PATCH 19/51] Updates to the instructions in the header comment of the fuzzer implementation. New test cases for the fuzzer. FossilOrigin-Name: bf1dc7907cf1a5c7e19b04fa1278b2089316c30a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/test_fuzzer.c | 39 ++++++++++++++++++++++++++------------- test/fuzzer1.test | 46 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 80 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index b0a676f806..c8032d0fb9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sway\sthe\sfuzzer\s(test_fuzzer.c)\sworks\sso\sthat\sit\sloads\sits\sconfiguration\sfrom\sa\sdatabase\stable. -D 2012-02-20T20:03:48.835 +C Updates\sto\sthe\sinstructions\sin\sthe\sheader\scomment\sof\sthe\sfuzzer\simplementation.\nNew\stest\scases\sfor\sthe\sfuzzer. +D 2012-02-20T22:44:12.628 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/test_config.c a036a69b550ebc477ab9ca2b37269201f888436e F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 6232d722a4ddb193035aa13a03796bf57d6c12fd -F src/test_fuzzer.c 010ee3d4122fd955d6f0db598f68d62f95d15fa9 +F src/test_fuzzer.c 2c0d96f94fdf7bfc2458dac9835c87b2086fdb67 F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -504,7 +504,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzer1.test 50a480932b91df9d61dd089f338e448991ab771e +F test/fuzzer1.test 830e260b10213d5dc6aadccc1fc4b0c7fefce7b8 F test/hook.test 5f3749de6462a6b87b4209b74adf7df5ac2df639 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test a7b8a0f43da81cd08645b7a710099ffe9ad1126b @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P cb5f5ebc563b8d3e47bc30b6dbb374bb91efd3ef -R 15ed6cfb8d0ad7f6ea31fbc888768f7f -U dan -Z d7a1a4dc789683dc31afc336a51e56bf +P 90b7b957f8933047fd2878048dfa3ec4891988b8 +R e208940f645a7d6e8e24937143961be4 +U drh +Z c02d350749266fd28db3de432a6902e8 diff --git a/manifest.uuid b/manifest.uuid index 8798c458c0..6ba7c0f0ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90b7b957f8933047fd2878048dfa3ec4891988b8 \ No newline at end of file +bf1dc7907cf1a5c7e19b04fa1278b2089316c30a \ No newline at end of file diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index 6953965243..bfc5b7a2cf 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -28,13 +28,13 @@ ** that consist of four columns). It does not matter what the columns are ** named. ** -** Each row in the fuzzer table represents a single character transformation. -** The left most column of the row (column 0) contains an integer value - -** the identifier of the ruleset to which the transformation rule belongs -** (see "MULTIPLE RULE SETS" below). The second column of the row (column 0) -** contains the input character or characters. The third column contains the -** output character or characters. And the fourth column contains the integer -** cost of making the transformation. For example: +** Each row in the fuzzer data table represents a single character +** transformation. The left most column of the row (column 0) contains an +** integer value - the identifier of the ruleset to which the transformation +** rule belongs (see "MULTIPLE RULE SETS" below). The second column of the +** row (column 0) contains the input character or characters. The third +** column contains the output character or characters. And the fourth column +** contains the integer cost of making the transformation. For example: ** ** CREATE TABLE f_data(ruleset, cFrom, cTo, Cost); ** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); @@ -46,9 +46,9 @@ ** above indicates that the cost of inserting a letter 'a' is 100. (All ** costs are integers. We recommend that costs be scaled so that the ** average cost is around 100.) The second INSERT statement creates a rule -** that the cost of that the cost of deleting a single letter 'b' is 87. -** The third and fourth INSERT statements mean that the cost of transforming -** a single letter "o" into the two-letter sequence "oe" is 38 and that the +** saying that the cost of deleting a single letter 'b' is 87. The third +** and fourth INSERT statements mean that the cost of transforming a +** single letter "o" into the two-letter sequence "oe" is 38 and that the ** cost of transforming "oe" back into "o" is 40. ** ** The contents of the fuzzer data table are loaded into main memory when @@ -58,6 +58,8 @@ ** If you do need to modify the contents of the fuzzer data table, it is ** recommended that the associated fuzzer table be dropped, the fuzzer data ** table edited, and the fuzzer table recreated within a single transaction. +** Alternatively, the fuzzer data table can be edited then the database +** connection can be closed and reopened. ** ** Once it has been created, the fuzzer table can be queried as follows: ** @@ -74,6 +76,9 @@ ** the one that is returned. In the example, the search is limited to ** strings with a total distance of less than 200. ** +** The fuzzer is a read-only table. Any attempt to DELETE, INSERT, or +** UPDATE on a fuzzer table will throw an error. +** ** It is important to put some kind of a limit on the fuzzer output. This ** can be either in the form of a LIMIT clause at the end of the query, ** or better, a "distance @@ -777,6 +788,7 @@ static fuzzer_stem *fuzzerNewStem( fuzzer_cost rBaseCost ){ fuzzer_stem *pNew; + fuzzer_rule *pRule; unsigned int h; pNew = sqlite3_malloc( sizeof(*pNew) + strlen(zWord) + 1 ); @@ -785,10 +797,11 @@ static fuzzer_stem *fuzzerNewStem( pNew->zBasis = (char*)&pNew[1]; pNew->nBasis = strlen(zWord); memcpy(pNew->zBasis, zWord, pNew->nBasis+1); - pNew->pRule = pCur->pVtab->pRule; - while( pNew->pRule && pNew->pRule->iRuleset!=pCur->iRuleset ){ - pNew->pRule = pNew->pRule->pNext; + pRule = pCur->pVtab->pRule; + while( pRule && pRule->iRuleset!=pCur->iRuleset ){ + pRule = pRule->pNext; } + pNew->pRule = pRule; pNew->n = -1; pNew->rBaseCost = pNew->rCostX = rBaseCost; h = fuzzerHash(pNew->zBasis); diff --git a/test/fuzzer1.test b/test/fuzzer1.test index cfa6b756d5..8e8d581a77 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -147,11 +147,21 @@ do_test fuzzer1-1.13 { WHERE word MATCH 'abcde' AND distance<=11 AND ruleset=1 } } {abcde 0 axcde 1 abcye 10 axcye 11} +do_test fuzzer1-1.14 { + catchsql {INSERT INTO f1 VALUES(1)} +} {1 {table f1 may not be modified}} +do_test fuzzer1-1.15 { + catchsql {DELETE FROM f1} +} {1 {table f1 may not be modified}} +do_test fuzzer1-1.16 { + catchsql {UPDATE f1 SET rowid=rowid+10000} +} {1 {table f1 may not be modified}} + do_test fuzzer1-2.0 { execsql { -- costs based on English letter frequencies - CREATE TEMP TABLE f2_rules(ruleset, cFrom, cTo, cost); + CREATE TEMP TABLE f2_rules(ruleset DEFAULT 0, cFrom, cTo, cost); INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','e',24); INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','o',47); INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('a','u',50); @@ -220,6 +230,18 @@ do_test fuzzer1-2.0 { INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('y','',100); INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('','z',120); INSERT INTO f2_rules(cFrom,cTo,cost) VALUES('z','',120); + INSERT INTO f2_rules(ruleset,cFrom,cTo,cost) + SELECT 1, cFrom, cTo, 100 FROM f2_rules WHERE ruleset=0; + INSERT INTO f2_rules(ruleset,cFrom,cTo,cost) + SELECT 2, cFrom, cTo, 200-cost FROM f2_rules WHERE ruleset=0; + INSERT INTO f2_rules(ruleset,cFrom,cTo,cost) + SELECT 3, cFrom, cTo, cost FROM f2_rules WHERE ruleset=0; + INSERT INTO f2_rules(ruleset,cFrom,cTo,cost) + VALUES(3, 'mallard','duck',50), + (3, 'duck', 'mallard', 50), + (3, 'rock', 'stone', 50), + (3, 'stone', 'rock', 50); + CREATE VIRTUAL TABLE temp.f2 USING fuzzer(f2_rules); @@ -1483,6 +1505,28 @@ do_test fuzzer1-2.3 { AND streetname.n>=f2.word AND streetname.n<=(f2.word || x'F7BFBFBF') } } {{tyler finley} trailer taymouth steelewood tallia tallu talwyn thelema} +do_test fuzzer1-2.4 { + execsql { + SELECT DISTINCT streetname.n + FROM f2 JOIN streetname + ON (streetname.n>=f2.word AND streetname.n<=(f2.word || 'zzzzzz')) + WHERE f2.word MATCH 'duck' + AND f2.distance<150 + AND f2.ruleset=3 + ORDER BY 1 + } +} {mallard {mallard cove} {mallard forest} {mallard grove} {mallard hill} {mallard park} {mallard ridge} {mallard view}} +do_test fuzzer1-2.5 { + execsql { + SELECT DISTINCT streetname.n + FROM f2 JOIN streetname + ON (streetname.n>=f2.word AND streetname.n<=(f2.word || 'zzzzzz')) + WHERE f2.word MATCH 'duck' + AND f2.distance<150 + AND f2.ruleset=2 + ORDER BY 1 + } +} {} forcedelete test.db2 do_execsql_test fuzzer1-4.1 { From 766348febe712ba06a6234bc56c0c50c4fd88d84 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Feb 2012 10:36:27 +0000 Subject: [PATCH 20/51] Add further test cases and minor fixes for the fuzzer. FossilOrigin-Name: 583dde93a9176ba4fff85241bafbbe4e5a6cc95f --- manifest | 16 ++--- manifest.uuid | 2 +- src/test_fuzzer.c | 109 +++++++++++++++++++++------- test/fuzzer1.test | 180 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 272 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index c8032d0fb9..b5477ec6ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\sinstructions\sin\sthe\sheader\scomment\sof\sthe\sfuzzer\simplementation.\nNew\stest\scases\sfor\sthe\sfuzzer. -D 2012-02-20T22:44:12.628 +C Add\sfurther\stest\scases\sand\sminor\sfixes\sfor\sthe\sfuzzer. +D 2012-02-21T10:36:27.146 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/test_config.c a036a69b550ebc477ab9ca2b37269201f888436e F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 6232d722a4ddb193035aa13a03796bf57d6c12fd -F src/test_fuzzer.c 2c0d96f94fdf7bfc2458dac9835c87b2086fdb67 +F src/test_fuzzer.c d557ab99e9f1c0d8ae7ac639b4c3d4babd279f90 F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -504,7 +504,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzer1.test 830e260b10213d5dc6aadccc1fc4b0c7fefce7b8 +F test/fuzzer1.test 098fc6dc6edcc8430d8e29a27eae6b9825d35470 F test/hook.test 5f3749de6462a6b87b4209b74adf7df5ac2df639 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test a7b8a0f43da81cd08645b7a710099ffe9ad1126b @@ -989,7 +989,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 90b7b957f8933047fd2878048dfa3ec4891988b8 -R e208940f645a7d6e8e24937143961be4 -U drh -Z c02d350749266fd28db3de432a6902e8 +P bf1dc7907cf1a5c7e19b04fa1278b2089316c30a +R e5d50c19b0286a351f7b9b6f103c6c32 +U dan +Z 5de28f986fce8d4352f35f6c3b70b1eb diff --git a/manifest.uuid b/manifest.uuid index 6ba7c0f0ec..fd44690269 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf1dc7907cf1a5c7e19b04fa1278b2089316c30a \ No newline at end of file +583dde93a9176ba4fff85241bafbbe4e5a6cc95f \ No newline at end of file diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index bfc5b7a2cf..eecea140e0 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -141,6 +141,12 @@ ** of the strings in the second or third column of the fuzzer data table ** is 50 bytes. The maximum cost on a rule is 1000. */ + +/* If SQLITE_DEBUG is not defined, disable assert statements. */ +#ifndef SQLITE_DEBUG +# define NDEBUG +#endif + #include "sqlite3.h" #include #include @@ -289,7 +295,7 @@ static int fuzzerLoadOneRule( fuzzer_rule **ppRule, /* OUT: New rule object */ char **pzErr /* OUT: Error message */ ){ - int iRuleset = sqlite3_column_int(pStmt, 0); + sqlite3_int64 iRuleset = sqlite3_column_int64(pStmt, 0); const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); int nCost = sqlite3_column_int(pStmt, 3); @@ -311,16 +317,21 @@ static int fuzzerLoadOneRule( } if( nCost<=0 || nCost>FUZZER_MX_COST ){ - *pzErr = sqlite3_mprintf("cost must be between 1 and %d", FUZZER_MX_COST); + *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d", + p->zClassName, FUZZER_MX_COST + ); rc = SQLITE_ERROR; }else if( nFrom>FUZZER_MX_LENGTH || nTo>FUZZER_MX_LENGTH ){ - *pzErr = sqlite3_mprintf("maximum string length is %d", FUZZER_MX_LENGTH); + *pzErr = sqlite3_mprintf("%s: maximum string length is %d", + p->zClassName, FUZZER_MX_LENGTH + ); rc = SQLITE_ERROR; }else if( iRuleset<0 || iRuleset>FUZZER_MX_RULEID ){ - *pzErr = sqlite3_mprintf( - "ruleset must be between 0 and %d", FUZZER_MX_RULEID); + *pzErr = sqlite3_mprintf("%s: ruleset must be between 0 and %d", + p->zClassName, FUZZER_MX_RULEID + ); rc = SQLITE_ERROR; }else{ @@ -335,7 +346,7 @@ static int fuzzerLoadOneRule( memcpy(pRule->zTo, zTo, nTo+1); pRule->nTo = nTo; pRule->rCost = nCost; - pRule->iRuleset = iRuleset; + pRule->iRuleset = (int)iRuleset; } } @@ -419,6 +430,59 @@ static int fuzzerLoadRules( return rc; } +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *fuzzerDequote(const char *zIn){ + int nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = strlen(zIn); + zOut = sqlite3_malloc(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, nIn+1); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInnCursor==0 ); + while( p->pRule ){ + fuzzer_rule *pRule = p->pRule; + p->pRule = pRule->pNext; + sqlite3_free(pRule); + } + sqlite3_free(p); + return SQLITE_OK; +} /* ** xConnect/xCreate method for the fuzzer module. Arguments are: @@ -453,15 +517,24 @@ static int fuzzerConnect( if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ + char *zTab; /* Dequoted name of fuzzer data table */ + memset(pNew, 0, sizeof(*pNew)); pNew->zClassName = (char*)&pNew[1]; memcpy(pNew->zClassName, zModule, nModule+1); - rc = fuzzerLoadRules(db, pNew, zDb, argv[3], pzErr); - if( rc==SQLITE_OK ){ - sqlite3_declare_vtab(db, "CREATE TABLE x(word, distance,ruleset)"); + zTab = fuzzerDequote(argv[3]); + if( zTab==0 ){ + rc = SQLITE_NOMEM; }else{ - sqlite3_free(pNew); + rc = fuzzerLoadRules(db, pNew, zDb, zTab, pzErr); + sqlite3_free(zTab); + } + + if( rc==SQLITE_OK ){ + sqlite3_declare_vtab(db, "CREATE TABLE x(word, distance, ruleset)"); + }else{ + fuzzerDisconnect((sqlite3_vtab *)pNew); pNew = 0; } } @@ -470,22 +543,6 @@ static int fuzzerConnect( *ppVtab = (sqlite3_vtab *)pNew; return rc; } -/* Note that for this virtual table, the xCreate and xConnect -** methods are identical. */ - -static int fuzzerDisconnect(sqlite3_vtab *pVtab){ - fuzzer_vtab *p = (fuzzer_vtab*)pVtab; - assert( p->nCursor==0 ); - while( p->pRule ){ - fuzzer_rule *pRule = p->pRule; - p->pRule = pRule->pNext; - sqlite3_free(pRule); - } - sqlite3_free(p); - return SQLITE_OK; -} -/* The xDisconnect and xDestroy methods are also the same */ - /* ** Open a new fuzzer cursor. diff --git a/test/fuzzer1.test b/test/fuzzer1.test index 8e8d581a77..89662c86fe 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -22,6 +22,8 @@ ifcapable !vtab { return } +set ::testprefix fuzzer1 + register_fuzzer_module db @@ -1538,4 +1540,182 @@ do_execsql_test fuzzer1-4.1 { SELECT word FROM f3 WHERE word MATCH 'ax' } {ax ay} +#------------------------------------------------------------------------- +# +# 1.5.1 - Check things work with a fuzzer data table name that requires +# quoting. Also that NULL entries in the "from" column of the +# data table are treated as zero length strings (''). +# +# 1.5.2 - Check that no-op rules (i.e. C->C) are ignored. Test NULL in +# the "to" column of a fuzzer data table. +# +# 1.5.3 - Test out-of-range values for the cost field of the data table. +# +# 1.5.4 - Test out-of-range values for the string fields of the data table. +# +# 1.5.5 - Test out-of-range values for the ruleset field of the data table. +# +do_execsql_test 5.1 { + CREATE TABLE "fuzzer [x] rules table"(a, b, c, d); + INSERT INTO "fuzzer [x] rules table" VALUES(0, NULL, 'abc', 10); + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); + SELECT word, distance FROM x WHERE word MATCH '123' LIMIT 4; +} {123 0 abc123 10 1abc23 10 12abc3 10} + +do_execsql_test 5.2 { + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES(0, 'x', NULL, 20); + INSERT INTO "fuzzer [x] rules table" VALUES(0, NULL, NULL, 10); + INSERT INTO "fuzzer [x] rules table" VALUES(0, 'x', 'x', 10); + + DROP TABLE x; + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); + + SELECT word, distance FROM x WHERE word MATCH 'xx'; +} {xx 0 x 20 {} 40} + +do_execsql_test 5.3.1 { + DROP TABLE IF EXISTS x; + INSERT INTO "fuzzer [x] rules table" VALUES(0, 'c', 'd', 1001); +} +do_catchsql_test 5.3.2 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: cost must be between 1 and 1000}} + +do_execsql_test 5.3.3 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES(0, 'd', 'c', 0); +} +do_catchsql_test 5.3.4 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: cost must be between 1 and 1000}} + +do_execsql_test 5.3.5 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES(0, 'd', 'c', -20); +} +do_catchsql_test 5.3.6 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: cost must be between 1 and 1000}} + +do_execsql_test 5.4.1 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES( + 0, 'x', '12345678901234567890123456789012345678901234567890', 2 + ); + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); + SELECT word FROM x WHERE word MATCH 'x'; +} {x 12345678901234567890123456789012345678901234567890} + +do_execsql_test 5.4.2 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES( + 0, 'x', '123456789012345678901234567890123456789012345678901', 2 + ); +} +do_catchsql_test 5.4.3 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: maximum string length is 50}} + +do_execsql_test 5.4.4 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES( + 0, '123456789012345678901234567890123456789012345678901', 'x', 2 + ); +} +do_catchsql_test 5.4.5 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: maximum string length is 50}} + +do_execsql_test 5.5.1 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES(-1, 'x', 'y', 2); +} +do_catchsql_test 5.5.2 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: ruleset must be between 0 and 2147483647}} + +do_execsql_test 5.5.3 { + DROP TABLE IF EXISTS x; + DELETE FROM "fuzzer [x] rules table"; + INSERT INTO "fuzzer [x] rules table" VALUES((1<<32)+100, 'x', 'y', 2); +} +do_catchsql_test 5.5.4 { + CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); +} {1 {fuzzer: ruleset must be between 0 and 2147483647}} + +#------------------------------------------------------------------------- +# This test uses a fuzzer table with many rules. There is one rule to +# map each possible two character string, where characters are lower-case +# letters used in the English language, to all other possible two character +# strings. In total, (26^4)-(26^2) mappings (the subtracted term represents +# the no-op mappings discarded automatically by the fuzzer). +# +# +do_execsql_test 6.1.1 { + DROP TABLE IF EXISTS x1; + DROP TABLE IF EXISTS x1_rules; + CREATE TABLE x1_rules(ruleset, cFrom, cTo, cost); +} +puts "This test is slow - perhaps around 7 seconds on an average pc" +do_test 6.1.2 { + set LETTERS {a b c d e f g h i j k l m n o p q r s t u v w x y z} + set cost 1 + db transaction { + foreach c1 $LETTERS { + foreach c2 $LETTERS { + foreach c3 $LETTERS { + foreach c4 $LETTERS { + db eval {INSERT INTO x1_rules VALUES(0, $c1||$c2, $c3||$c4, $cost)} + set cost [expr ($cost%1000) + 1] + } + } + } + } + db eval {UPDATE x1_rules SET cost = 20 WHERE cost<20 AND cFrom!='xx'} + } +} {} + +do_execsql_test 6.2 { + SELECT count(*) FROM x1_rules WHERE cTo!=cFrom; +} [expr 26*26*26*26 - 26*26] + +do_execsql_test 6.2.1 { + CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); + SELECT word FROM x1 WHERE word MATCH 'xx' LIMIT 10; +} {xx hw hx hy hz ia ib ic id ie} +do_execsql_test 6.2.2 { + SELECT cTo FROM x1_rules WHERE cFrom='xx' + ORDER BY cost asc, rowid asc LIMIT 9; +} {hw hx hy hz ia ib ic id ie} + + +#------------------------------------------------------------------------- +# Test using different types of quotes with CREATE VIRTUAL TABLE +# arguments. +# +do_execsql_test 7.1 { + CREATE TABLE [x2 "rules] (a, b, c, d); + INSERT INTO [x2 "rules] VALUES(0, 'a', 'b', 5); +} +foreach {tn sql} { + 1 { CREATE VIRTUAL TABLE x2 USING fuzzer( [x2 "rules] ) } + 2 { CREATE VIRTUAL TABLE x2 USING fuzzer( "x2 ""rules" ) } + 3 { CREATE VIRTUAL TABLE x2 USING fuzzer( 'x2 "rules' ) } + 4 { CREATE VIRTUAL TABLE x2 USING fuzzer( `x2 "rules` ) } +} { + do_execsql_test 7.2.$tn.1 { DROP TABLE IF EXISTS x2 } + do_execsql_test 7.2.$tn.2 $sql + do_execsql_test 7.2.$tn.3 { + SELECT word FROM x2 WHERE word MATCH 'aaa' + } {aaa baa aba aab bab abb bba bbb} +} + finish_test + From 4965ebceb7d233fc8db57c0e7411c43301a5a3bc Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Feb 2012 14:11:12 +0000 Subject: [PATCH 21/51] Further fuzzer test cases. Fix a case in the fuzzer where an error code was being dropped. FossilOrigin-Name: 8b77d3953f082e6fa42e42fe903ea9eddf99dd91 --- manifest | 15 +++--- manifest.uuid | 2 +- src/test_fuzzer.c | 5 +- test/fuzzer1.test | 110 +++++++++++++++++++++++++++++++++++++++++- test/fuzzerfault.test | 47 ++++++++++++++++++ 5 files changed, 168 insertions(+), 11 deletions(-) create mode 100644 test/fuzzerfault.test diff --git a/manifest b/manifest index b5477ec6ed..833e77ac46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stest\scases\sand\sminor\sfixes\sfor\sthe\sfuzzer. -D 2012-02-21T10:36:27.146 +C Further\sfuzzer\stest\scases.\sFix\sa\scase\sin\sthe\sfuzzer\swhere\san\serror\scode\swas\sbeing\sdropped. +D 2012-02-21T14:11:12.123 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/test_config.c a036a69b550ebc477ab9ca2b37269201f888436e F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 6232d722a4ddb193035aa13a03796bf57d6c12fd -F src/test_fuzzer.c d557ab99e9f1c0d8ae7ac639b4c3d4babd279f90 +F src/test_fuzzer.c 5d716ce83ee678471dbf91a05c33ab05f115ed61 F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -504,7 +504,8 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzer1.test 098fc6dc6edcc8430d8e29a27eae6b9825d35470 +F test/fuzzer1.test 8fb031a789f68c29236bf90e31919ebbf94cb730 +F test/fuzzerfault.test 00fba714b2c52603c532cbf336b4901252e0e48d F test/hook.test 5f3749de6462a6b87b4209b74adf7df5ac2df639 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test a7b8a0f43da81cd08645b7a710099ffe9ad1126b @@ -989,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P bf1dc7907cf1a5c7e19b04fa1278b2089316c30a -R e5d50c19b0286a351f7b9b6f103c6c32 +P 583dde93a9176ba4fff85241bafbbe4e5a6cc95f +R 5eaa36a3b096a39cabce5b0dfd058e68 U dan -Z 5de28f986fce8d4352f35f6c3b70b1eb +Z 4089608a3cec5128a943f2b568b6cca4 diff --git a/manifest.uuid b/manifest.uuid index fd44690269..3d327dbee9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -583dde93a9176ba4fff85241bafbbe4e5a6cc95f \ No newline at end of file +8b77d3953f082e6fa42e42fe903ea9eddf99dd91 \ No newline at end of file diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index eecea140e0..c7d4cfc218 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -532,8 +532,9 @@ static int fuzzerConnect( } if( rc==SQLITE_OK ){ - sqlite3_declare_vtab(db, "CREATE TABLE x(word, distance, ruleset)"); - }else{ + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(word,distance,ruleset)"); + } + if( rc!=SQLITE_OK ){ fuzzerDisconnect((sqlite3_vtab *)pNew); pNew = 0; } diff --git a/test/fuzzer1.test b/test/fuzzer1.test index 89662c86fe..97209c3273 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -1695,7 +1695,6 @@ do_execsql_test 6.2.2 { ORDER BY cost asc, rowid asc LIMIT 9; } {hw hx hy hz ia ib ic id ie} - #------------------------------------------------------------------------- # Test using different types of quotes with CREATE VIRTUAL TABLE # arguments. @@ -1717,5 +1716,114 @@ foreach {tn sql} { } {aaa baa aba aab bab abb bba bbb} } +#------------------------------------------------------------------------- +# Test using a fuzzer table in different contexts. +# +do_execsql_test 8.1 { + CREATE TABLE x3_rules(rule_set, cFrom, cTo, cost); + INSERT INTO x3_rules VALUES(2, 'a', 'x', 10); + INSERT INTO x3_rules VALUES(2, 'a', 'y', 9); + INSERT INTO x3_rules VALUES(2, 'a', 'z', 8); + CREATE VIRTUAL TABLE x3 USING fuzzer(x3_rules); +} + +do_execsql_test 8.2.1 { + SELECT cFrom, cTo, word + FROM x3_rules CROSS JOIN x3 + WHERE word MATCH 'a' AND cost=distance AND ruleset=2; +} {a x x a y y a z z} + +do_execsql_test 8.2.2 { + SELECT cFrom, cTo, word + FROM x3 CROSS JOIN x3_rules + WHERE word MATCH 'a' AND cost=distance AND ruleset=2; +} {a z z a y y a x x} + +do_execsql_test 8.2.3 { + SELECT cFrom, cTo, word + FROM x3_rules, x3 + WHERE word MATCH 'a' AND cost=distance AND ruleset=2; +} {a z z a y y a x x} + +do_execsql_test 8.2.4 { + SELECT cFrom, cTo, word + FROM x3, x3_rules + WHERE word MATCH 'a' AND cost=distance AND ruleset=2; +} {a z z a y y a x x} + +do_execsql_test 8.2.5 { + CREATE INDEX i1 ON x3_rules(cost); + SELECT cFrom, cTo, word + FROM x3_rules, x3 + WHERE word MATCH 'a' AND cost=distance AND ruleset=2; +} {a z z a y y a x x} + +do_execsql_test 8.2.5 { + SELECT word FROM x3_rules, x3 WHERE word MATCH x3_rules.cFrom AND ruleset=2; +} {a z y x a z y x a z y x} + +do_execsql_test 8.2.6 { + SELECT word FROM x3_rules, x3 + WHERE word MATCH x3_rules.cFrom + AND ruleset=2 + AND x3_rules.cost=8; +} {a z y x} + +do_execsql_test 8.2.7 { + CREATE TABLE t1(a, b); + CREATE INDEX i2 ON t1(b); + SELECT word, distance FROM x3, t1 + WHERE x3.word MATCH t1.a AND ruleset=2 AND distance=t1.b; +} {} + +do_execsql_test 8.2.8 { + INSERT INTO x3_rules VALUES(1, 'a', 't', 5); + INSERT INTO x3_rules VALUES(1, 'a', 'u', 4); + INSERT INTO x3_rules VALUES(1, 'a', 'v', 3); + DROP TABLE x3; + CREATE VIRTUAL TABLE x3 USING fuzzer(x3_rules); + SELECT * FROM x3_rules; +} { + 2 a x 10 + 2 a y 9 + 2 a z 8 + 1 a t 5 + 1 a u 4 + 1 a v 3 +} + +do_catchsql_test 8.2.9 { + SELECT word FROM x3 WHERE ruleset=2 AND word MATCH 'a' AND WORD MATCH 'b'; +} {1 {unable to use function MATCH in the requested context}} + +do_execsql_test 8.2.10 { + SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' +} {a v u t} + +# The term "ruleset<=1" is not handled by the fuzzer module. Instead, it +# is handled by SQLite, which assumes that all rows have a NULL value in +# the ruleset column. Since NULL<=1 is never true, this query returns +# no rows. +do_execsql_test 8.2.11 { + SELECT word FROM x3 WHERE ruleset<=1 AND word MATCH 'a' +} {} + +do_execsql_test 8.2.12 { + SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' ORDER BY distance ASC; +} {a v u t} + +do_execsql_test 8.2.13 { + SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' ORDER BY distance DESC; +} {t u v a} + +do_execsql_test 8.2.13 { + SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' ORDER BY word ASC; +} {a t u v} + +do_execsql_test 8.2.14 { + SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' ORDER BY word DESC; +} {v u t a} + + finish_test diff --git a/test/fuzzerfault.test b/test/fuzzerfault.test new file mode 100644 index 0000000000..7ef26af1f6 --- /dev/null +++ b/test/fuzzerfault.test @@ -0,0 +1,47 @@ +# 2012 February 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for TCL interface to the +# SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !vtab { finish_test ; return } +set ::testprefix fuzzerfault + +register_fuzzer_module db + +do_test 1-pre1 { + execsql { + CREATE TABLE x1_rules(ruleset, cFrom, cTo, cost); + INSERT INTO x1_rules VALUES(0, 'a', 'b', 1); + INSERT INTO x1_rules VALUES(0, 'a', 'c', 2); + INSERT INTO x1_rules VALUES(0, 'a', 'd', 3); + } + faultsim_save_and_close +} {} + +do_faultsim_test 1 -faults oom-t* -prep { + faultsim_restore_and_reopen + register_fuzzer_module db +} -body { + execsql { + CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); + SELECT word FROM x1 WHERE word MATCH 'xax'; + } +} -test { + faultsim_test_result {0 {xax xbx xcx xdx}} \ + {1 {vtable constructor failed: x1}} +} + + + +finish_test From 99c7d468a786d502b2c34514688c0e8105fc0c20 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Feb 2012 18:00:20 +0000 Subject: [PATCH 22/51] Prevent the fuzzer from ever returning a string longer than 100 bytes. FossilOrigin-Name: 969095ca3067d60f4d3a0a4ad419541cb8b146db --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/test_fuzzer.c | 60 +++++++++++++++++++++++++++++++++---------- test/fuzzer1.test | 43 +++++++++++++++++++++++++++++-- test/fuzzerfault.test | 49 +++++++++++++++++++++++++++++++++-- 5 files changed, 143 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 833e77ac46..b23524e5b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sfuzzer\stest\scases.\sFix\sa\scase\sin\sthe\sfuzzer\swhere\san\serror\scode\swas\sbeing\sdropped. -D 2012-02-21T14:11:12.123 +C Prevent\sthe\sfuzzer\sfrom\sever\sreturning\sa\sstring\slonger\sthan\s100\sbytes. +D 2012-02-21T18:00:20.124 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/test_config.c a036a69b550ebc477ab9ca2b37269201f888436e F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 6232d722a4ddb193035aa13a03796bf57d6c12fd -F src/test_fuzzer.c 5d716ce83ee678471dbf91a05c33ab05f115ed61 +F src/test_fuzzer.c e81b8984e9649499c45b080b60ebe4c1f6cff8d0 F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -504,8 +504,8 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzer1.test 8fb031a789f68c29236bf90e31919ebbf94cb730 -F test/fuzzerfault.test 00fba714b2c52603c532cbf336b4901252e0e48d +F test/fuzzer1.test 69cf1036b92fd3b8e1fd65bef4d7ee3f085c28fb +F test/fuzzerfault.test ff2282c81797b6a355f0748d8b54c7287c5d2b25 F test/hook.test 5f3749de6462a6b87b4209b74adf7df5ac2df639 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test a7b8a0f43da81cd08645b7a710099ffe9ad1126b @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 583dde93a9176ba4fff85241bafbbe4e5a6cc95f -R 5eaa36a3b096a39cabce5b0dfd058e68 +P 8b77d3953f082e6fa42e42fe903ea9eddf99dd91 +R eac7c759ba0fcf7d16f2f5a90574c01e U dan -Z 4089608a3cec5128a943f2b568b6cca4 +Z 5ce06301e86d5442c836392119f93673 diff --git a/manifest.uuid b/manifest.uuid index 3d327dbee9..13c72154f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b77d3953f082e6fa42e42fe903ea9eddf99dd91 \ No newline at end of file +969095ca3067d60f4d3a0a4ad419541cb8b146db \ No newline at end of file diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index c7d4cfc218..66d0bd08d4 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -180,9 +180,10 @@ typedef int fuzzer_ruleid; /* ** Limits */ -#define FUZZER_MX_LENGTH 50 /* Maximum length of a search string */ +#define FUZZER_MX_LENGTH 50 /* Maximum length of a rule string */ #define FUZZER_MX_RULEID 2147483647 /* Maximum rule ID */ #define FUZZER_MX_COST 1000 /* Maximum single-rule cost */ +#define FUZZER_MX_OUTPUT_LENGTH 100 /* Maximum length of an output string */ /* @@ -612,8 +613,8 @@ static int fuzzerRender( int *pnBuf /* Size of the buffer */ ){ const fuzzer_rule *pRule = pStem->pRule; - int n; - char *z; + int n; /* Size of output term without nul-term */ + char *z; /* Buffer to assemble output term in */ n = pStem->nBasis + pRule->nTo - pRule->nFrom; if( (*pnBuf)nTo], &pStem->zBasis[n+pRule->nFrom], pStem->nBasis-n-pRule->nFrom+1); } + + assert( z[pStem->nBasis + pRule->nTo - pRule->nFrom]==0 ); return SQLITE_OK; } @@ -699,6 +702,25 @@ static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ return pLookup!=0; } +/* +** If argument pRule is NULL, this function returns false. +** +** Otherwise, it returns true if rule pRule should be skipped. A rule +** should be skipped if it does not belong to rule-set iRuleset, or if +** applying it to stem pStem would create a string longer than +** FUZZER_MX_OUTPUT_LENGTH bytes. +*/ +static int fuzzerSkipRule( + const fuzzer_rule *pRule, /* Determine whether or not to skip this */ + fuzzer_stem *pStem, /* Stem rule may be applied to */ + int iRuleset /* Rule-set used by the current query */ +){ + return pRule && ( + (pRule->iRuleset!=iRuleset) + || (pStem->nBasis + pRule->nTo - pRule->nFrom)>FUZZER_MX_OUTPUT_LENGTH + ); +} + /* ** Advance a fuzzer_stem to its next value. Return 0 if there are ** no more values that can be generated by this fuzzer_stem. Return @@ -706,7 +728,6 @@ static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ */ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ const fuzzer_rule *pRule; - const int iSet = pCur->iRuleset; while( (pRule = pStem->pRule)!=0 ){ assert( pRule==&pCur->nullRule || pRule->iRuleset==pCur->iRuleset ); while( pStem->n < pStem->nBasis - pRule->nFrom ){ @@ -726,7 +747,7 @@ static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ pStem->n = -1; do{ pRule = pRule->pNext; - }while( pRule && pRule->iRuleset!=iSet ); + }while( fuzzerSkipRule(pRule, pStem, pCur->iRuleset) ); pStem->pRule = pRule; if( pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; } @@ -856,7 +877,7 @@ static fuzzer_stem *fuzzerNewStem( pNew->nBasis = strlen(zWord); memcpy(pNew->zBasis, zWord, pNew->nBasis+1); pRule = pCur->pVtab->pRule; - while( pRule && pRule->iRuleset!=pCur->iRuleset ){ + while( fuzzerSkipRule(pRule, pNew, pCur->iRuleset) ){ pRule = pRule->pNext; } pNew->pRule = pRule; @@ -906,7 +927,10 @@ static int fuzzerNext(sqlite3_vtab_cursor *cur){ ** stem list is the next lowest cost word. */ while( (pStem = pCur->pStem)!=0 ){ - if( fuzzerAdvance(pCur, pStem) ){ + int res = fuzzerAdvance(pCur, pStem); + if( res<0 ){ + return SQLITE_NOMEM; + }else if( res>0 ){ pCur->pStem = 0; pStem = fuzzerInsert(pCur, pStem); if( (rc = fuzzerSeen(pCur, pStem))!=0 ){ @@ -944,7 +968,7 @@ static int fuzzerFilter( int argc, sqlite3_value **argv ){ fuzzer_cursor *pCur = (fuzzer_cursor *)pVtabCursor; - const char *zWord = 0; + const char *zWord = ""; fuzzer_stem *pStem; int idx; @@ -963,17 +987,25 @@ static int fuzzerFilter( pCur->iRuleset = (fuzzer_cost)sqlite3_value_int(argv[idx]); idx++; } - if( zWord==0 ) zWord = ""; - pCur->pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); - if( pStem==0 ) return SQLITE_NOMEM; pCur->nullRule.pNext = pCur->pVtab->pRule; pCur->nullRule.rCost = 0; pCur->nullRule.nFrom = 0; pCur->nullRule.nTo = 0; pCur->nullRule.zFrom = ""; - pStem->pRule = &pCur->nullRule; - pStem->n = pStem->nBasis; pCur->iRowid = 1; + assert( pCur->pStem==0 ); + + /* If the query term is longer than FUZZER_MX_OUTPUT_LENGTH bytes, this + ** query will return zero rows. */ + if( strlen(zWord)pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); + if( pStem==0 ) return SQLITE_NOMEM; + pStem->pRule = &pCur->nullRule; + pStem->n = pStem->nBasis; + }else{ + pCur->rLimit = 0; + } + return SQLITE_OK; } @@ -1156,7 +1188,7 @@ static int register_fuzzer_module( Tcl_WrongNumArgs(interp, 1, objv, "DB"); return TCL_ERROR; } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + getDbPointer(interp, Tcl_GetString(objv[1]), &db); fuzzer_register(db); return TCL_OK; } diff --git a/test/fuzzer1.test b/test/fuzzer1.test index 97209c3273..dc8b445ee9 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -24,10 +24,13 @@ ifcapable !vtab { set ::testprefix fuzzer1 +# Test of test code. Only here to make the coverage metric better. +do_test 0.1 { + list [catch { register_fuzzer_module a b c } msg] $msg +} {1 {wrong # args: should be "register_fuzzer_module DB"}} + register_fuzzer_module db - - # Check configuration errors. # do_catchsql_test fuzzer1-1.1 { @@ -1824,6 +1827,42 @@ do_execsql_test 8.2.14 { SELECT word FROM x3 WHERE ruleset=1 AND word MATCH 'a' ORDER BY word DESC; } {v u t a} +#------------------------------------------------------------------------- +# +do_execsql_test 9.1 { + CREATE TABLE x4_rules(a, b, c, d); + INSERT INTO x4_rules VALUES(0, 'a', 'b', 10); + INSERT INTO x4_rules VALUES(0, 'a', 'c', 11); + INSERT INTO x4_rules VALUES(0, 'bx', 'zz', 20); + INSERT INTO x4_rules VALUES(0, 'cx', 'yy', 15); + INSERT INTO x4_rules VALUES(0, 'zz', '!!', 50); + CREATE VIRTUAL TABLE x4 USING fuzzer(x4_rules); +} + +do_execsql_test 9.2 { + SELECT word, distance FROM x4 WHERE word MATCH 'ax'; +} {ax 0 bx 10 cx 11 yy 26 zz 30 !! 80} + + +do_execsql_test 10.1 { + CREATE TABLE x5_rules(a, b, c, d); + CREATE VIRTUAL TABLE x5 USING fuzzer(x5_rules); +} + +do_execsql_test 10.2 { + SELECT word, distance FROM x5 WHERE word MATCH + 'aaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaa' || + 'aaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaa' || + 'aaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaaXaaaaaaaaa' +} {} + +do_execsql_test 10.3 { + INSERT INTO x5_rules VALUES(0, 'a', '0.1.2.3.4.5.6.7.8.9.a', 1); + DROP TABLE x5; + CREATE VIRTUAL TABLE x5 USING fuzzer(x5_rules); + SELECT length(word) FROM x5 WHERE word MATCH 'a' LIMIT 50; +} {1 21 41 61 81} finish_test + diff --git a/test/fuzzerfault.test b/test/fuzzerfault.test index 7ef26af1f6..067da7f5fc 100644 --- a/test/fuzzerfault.test +++ b/test/fuzzerfault.test @@ -28,8 +28,7 @@ do_test 1-pre1 { } faultsim_save_and_close } {} - -do_faultsim_test 1 -faults oom-t* -prep { +do_faultsim_test 1 -prep { faultsim_restore_and_reopen register_fuzzer_module db } -body { @@ -42,6 +41,52 @@ do_faultsim_test 1 -faults oom-t* -prep { {1 {vtable constructor failed: x1}} } +do_test 2-pre1 { + faultsim_delete_and_reopen + register_fuzzer_module db + execsql { + CREATE TABLE x2_rules(ruleset, cFrom, cTo, cost); + INSERT INTO x2_rules VALUES(0, 'a', 'x', 1); + INSERT INTO x2_rules VALUES(0, 'b', 'x', 2); + INSERT INTO x2_rules VALUES(0, 'c', 'x', 3); + CREATE VIRTUAL TABLE x2 USING fuzzer(x2_rules); + } + faultsim_save_and_close +} {} + +do_faultsim_test 2 -prep { + faultsim_restore_and_reopen + register_fuzzer_module db +} -body { + execsql { + SELECT count(*) FROM x2 WHERE word MATCH 'abc'; + } +} -test { + faultsim_test_result {0 8} {1 {vtable constructor failed: x2}} +} + +do_test 3-pre1 { + faultsim_delete_and_reopen + execsql { + CREATE TABLE x1_rules(ruleset, cFrom, cTo, cost); + INSERT INTO x1_rules VALUES(0, 'a', + '123456789012345678901234567890a1234567890123456789', 10 + ); + } + faultsim_save_and_close +} {} + +do_faultsim_test 3 -prep { + faultsim_restore_and_reopen + register_fuzzer_module db +} -body { + execsql { + CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); + SELECT count(*) FROM (SELECT * FROM x1 WHERE word MATCH 'a' LIMIT 2); + } +} -test { + faultsim_test_result {0 2} {1 {vtable constructor failed: x1}} +} finish_test From cebeee5dcb74262efff4c4f608aa08a2835bdcb9 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Feb 2012 18:35:42 +0000 Subject: [PATCH 23/51] Avoid redefining NDEBUG if the fuzzer is included in an amalagmated source file. FossilOrigin-Name: f8ac826f95ae6b7b981e4765d2e65b6839d0eff9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_fuzzer.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b23524e5b7..42136690f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Prevent\sthe\sfuzzer\sfrom\sever\sreturning\sa\sstring\slonger\sthan\s100\sbytes. -D 2012-02-21T18:00:20.124 +C Avoid\sredefining\sNDEBUG\sif\sthe\sfuzzer\sis\sincluded\sin\san\samalagmated\ssource\sfile. +D 2012-02-21T18:35:42.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/test_config.c a036a69b550ebc477ab9ca2b37269201f888436e F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c 6232d722a4ddb193035aa13a03796bf57d6c12fd -F src/test_fuzzer.c e81b8984e9649499c45b080b60ebe4c1f6cff8d0 +F src/test_fuzzer.c 3703a190bd79a43e5f097d59c73ab38961d14872 F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 8b77d3953f082e6fa42e42fe903ea9eddf99dd91 -R eac7c759ba0fcf7d16f2f5a90574c01e -U dan -Z 5ce06301e86d5442c836392119f93673 +P 969095ca3067d60f4d3a0a4ad419541cb8b146db +R 6f6307ffcfea232f8dcb8699efa50953 +U drh +Z 053a16dcdcfeef74bca7f647480c806f diff --git a/manifest.uuid b/manifest.uuid index 13c72154f0..85599c25d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -969095ca3067d60f4d3a0a4ad419541cb8b146db \ No newline at end of file +f8ac826f95ae6b7b981e4765d2e65b6839d0eff9 \ No newline at end of file diff --git a/src/test_fuzzer.c b/src/test_fuzzer.c index 66d0bd08d4..60d56ee1ea 100644 --- a/src/test_fuzzer.c +++ b/src/test_fuzzer.c @@ -143,7 +143,7 @@ */ /* If SQLITE_DEBUG is not defined, disable assert statements. */ -#ifndef SQLITE_DEBUG +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG #endif From 06fd5d63a4c562ac8a8bdf6119182f4d315e3248 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2012 14:45:19 +0000 Subject: [PATCH 24/51] Begin adding support for SQLITE_FCNTL_PRAGMA. FossilOrigin-Name: 5be07904baf6e5b8b99ca6f4bb6ddec973e59cd4 --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/pragma.c | 22 +++++++++++++++++++--- src/sqlite.h.in | 10 ++++++++++ src/test_vfstrace.c | 6 ++++++ 5 files changed, 47 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 42136690f4..a4bb376dd7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sredefining\sNDEBUG\sif\sthe\sfuzzer\sis\sincluded\sin\san\samalagmated\ssource\sfile. -D 2012-02-21T18:35:42.365 +C Begin\sadding\ssupport\sfor\sSQLITE_FCNTL_PRAGMA. +D 2012-02-22T14:45:19.357 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c b30b1c35908346ecc43d8d9d17f2ddf6817f8f60 -F src/pragma.c 350f59843f4ec4fca5dc63d497caf6433096bbdd +F src/pragma.c 4dfd5b263d459559cbecb877dc5f1e0fc822cc04 F src/prepare.c ec4989f7f480544bdc4192fe663470d2a2d7d61e F src/printf.c 7ffb4ebb8b341f67e049695ba031da717b3d2699 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -182,7 +182,7 @@ F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 232283a2e60d91cbd9a5ddf2f6f7ecf53d590075 F src/shell.c aa28f117033ba3e44b5eaaf2ad572222bcdfd66e -F src/sqlite.h.in 4f3230be44e084a1e98f91bb031b922f33dcf3f8 +F src/sqlite.h.in d51a37c14123717dec954c848f51fd635620ba00 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 F src/sqliteInt.h ea24b5f2d2e3ae7e0c1126a78c0e025ed4e98f4b F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d @@ -231,7 +231,7 @@ F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 F src/test_vfs.c 07157a0bbfe161cb5e32cad2079abd26cd611c4b -F src/test_vfstrace.c 065c7270a614254b2c68fbc7ba8d1fb1d5cbc823 +F src/test_vfstrace.c 374f6807d76e4899f51315469471b85db2703c2a F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 @@ -990,7 +990,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 969095ca3067d60f4d3a0a4ad419541cb8b146db -R 6f6307ffcfea232f8dcb8699efa50953 +P f8ac826f95ae6b7b981e4765d2e65b6839d0eff9 +R f3e55ac02da0d4d13eff7250cbdd55d7 +T *branch * file-control-pragma +T *sym-file-control-pragma * +T -sym-trunk * U drh -Z 053a16dcdcfeef74bca7f647480c806f +Z afe420533dd0a9b4e47e7db76c26bdaa diff --git a/manifest.uuid b/manifest.uuid index 85599c25d2..fcbf89b7a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8ac826f95ae6b7b981e4765d2e65b6839d0eff9 \ No newline at end of file +5be07904baf6e5b8b99ca6f4bb6ddec973e59cd4 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 187f9afe72..e449baa8fb 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -312,9 +312,12 @@ void sqlite3Pragma( const char *zDb = 0; /* The database name */ Token *pId; /* Pointer to token */ int iDb; /* Database index for */ - sqlite3 *db = pParse->db; - Db *pDb; - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); + const char *aFcntl[3]; /* Argument to SQLITE_FCNTL_PRAGMA */ + int rc; /* return value form SQLITE_FCNTL_PRAGMA */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* The specific database being pragmaed */ + Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); /* Prepared statement */ + if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); pParse->nMem = 2; @@ -345,6 +348,19 @@ void sqlite3Pragma( if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ goto pragma_out; } + + /* Send an SQLITE_FCNTL_PRAGMA file-control to the underlying VFS + ** connection. If it returns SQLITE_OK, then assume that the VFS + ** handled the pragma and generate a no-op prepared statement. + */ + aFcntl[0] = zLeft; + aFcntl[1] = zRight; + aFcntl[2] = 0; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + if( rc==SQLITE_OK ){ + zLeft = ""; + } + #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 6d2173d615..1bd1b08685 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -797,6 +797,15 @@ struct sqlite3_io_methods { ** do anything. Callers should initialize the char* variable to a NULL ** pointer in case this file-control is not implemented. This file-control ** is intended for diagnostic use only. +** +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** file control is sent to the open [sqlite3_file] object corresponding +** to the database file to which the pragma statement refers. ^If the +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [PRAGMA] processing continues. ^However, if the [SQLITE_FCNTL_PRAGMA] +** file control returns [SQLITE_OK], then the parser assumes that the +** VFS has handled the pragma itself and the parser generates a no-op +** prepared statement. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 @@ -811,6 +820,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_OVERWRITE 11 #define SQLITE_FCNTL_VFSNAME 12 #define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 +#define SQLITE_FCNTL_PRAGMA 14 /* ** CAPI3REF: Mutex Handle diff --git a/src/test_vfstrace.c b/src/test_vfstrace.c index 62577207b9..a1fa6ff442 100644 --- a/src/test_vfstrace.c +++ b/src/test_vfstrace.c @@ -476,6 +476,12 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ case SQLITE_FCNTL_OVERWRITE: zOp = "OVERWRITE"; break; case SQLITE_FCNTL_VFSNAME: zOp = "VFSNAME"; break; case 0xca093fa0: zOp = "DB_UNCHANGED"; break; + case SQLITE_FCNTL_PRAGMA: { + const char *const* a = (const char*const*)pArg; + sqlite3_snprintf(sizeof(zBuf), zBuf, "PRAGMA,[%s,%s]",a[0],a[1]); + zOp = zBuf; + break; + } default: { sqlite3_snprintf(sizeof zBuf, zBuf, "%d", op); zOp = zBuf; From 3fa9730273ec2979e46fb6c0c67a62181435db01 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2012 16:58:36 +0000 Subject: [PATCH 25/51] Change the SQLITE_FCNTL_PRAGMA file-control so that it can return a string value. FossilOrigin-Name: fd8d7440277b17f57a6dc0a210f42adf1fa2dcd2 --- manifest | 27 ++++++++++++--------------- manifest.uuid | 2 +- src/os_unix.c | 8 ++++++++ src/pragma.c | 20 ++++++++++++++------ src/sqlite.h.in | 9 +++++---- src/sqliteInt.h | 2 +- src/test_vfstrace.c | 6 +++++- src/util.c | 2 +- test/attach.test | 11 +++++++++++ 9 files changed, 58 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index a4bb376dd7..46df865eb6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\sadding\ssupport\sfor\sSQLITE_FCNTL_PRAGMA. -D 2012-02-22T14:45:19.357 +C Change\sthe\sSQLITE_FCNTL_PRAGMA\sfile-control\sso\sthat\sit\scan\sreturn\sa\sstring\nvalue. +D 2012-02-22T16:58:36.697 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 +F src/os_unix.c 497e355ab617a875bf45f4d58bacca82952339d5 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a @@ -174,7 +174,7 @@ F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c b30b1c35908346ecc43d8d9d17f2ddf6817f8f60 -F src/pragma.c 4dfd5b263d459559cbecb877dc5f1e0fc822cc04 +F src/pragma.c 6838f56b049840d015fc9b335ab4118479bdaac9 F src/prepare.c ec4989f7f480544bdc4192fe663470d2a2d7d61e F src/printf.c 7ffb4ebb8b341f67e049695ba031da717b3d2699 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -182,9 +182,9 @@ F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 232283a2e60d91cbd9a5ddf2f6f7ecf53d590075 F src/shell.c aa28f117033ba3e44b5eaaf2ad572222bcdfd66e -F src/sqlite.h.in d51a37c14123717dec954c848f51fd635620ba00 +F src/sqlite.h.in 5119a5c169c531d819b790957eebc28c4fc7acf6 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h ea24b5f2d2e3ae7e0c1126a78c0e025ed4e98f4b +F src/sqliteInt.h b013dab7d43fb67c3ca2f0253d7863abb37e233c F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 4568e72dfd36b6a5911f93457364deb072e0b03a F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -231,14 +231,14 @@ F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 F src/test_vfs.c 07157a0bbfe161cb5e32cad2079abd26cd611c4b -F src/test_vfstrace.c 374f6807d76e4899f51315469471b85db2703c2a +F src/test_vfstrace.c f20a32a5fd46feb59a93d2cf650f1ff92da3872b F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 -F src/util.c 9e07bd67dfafe9c75b1da78c87ba030cebbb5388 +F src/util.c 906731099c4397bf8adf3fa90a833355e7472af0 F src/vacuum.c 0c0ba2242355c6048d65e2b333abe0f7c06348fa F src/vdbe.c 21c17c43e579789959efea50f100de60efbb955c F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb @@ -275,7 +275,7 @@ F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 F test/async4.test 1787e3952128aa10238bf39945126de7ca23685a F test/async5.test 0dd8701bd588bf6e70c2557a22ae3f22b2567b4c -F test/attach.test 0d112b7713611fdf0340260192749737135fda5f +F test/attach.test 064911a24b6692759c661b981ed953c5fd9f441a F test/attach2.test e54436ed956d3d88bdee61221da59bf3935a0966 F test/attach3.test d89ccfe4fe6e2b5e368d480fcdfe4b496c54cf4e F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c @@ -990,10 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P f8ac826f95ae6b7b981e4765d2e65b6839d0eff9 -R f3e55ac02da0d4d13eff7250cbdd55d7 -T *branch * file-control-pragma -T *sym-file-control-pragma * -T -sym-trunk * +P 5be07904baf6e5b8b99ca6f4bb6ddec973e59cd4 +R f8a789f52fcbc32c2be7b7547e20bad5 U drh -Z afe420533dd0a9b4e47e7db76c26bdaa +Z 2d0e9a9b9d36c2e692d7eed74ee06028 diff --git a/manifest.uuid b/manifest.uuid index fcbf89b7a6..90e28dbbf0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5be07904baf6e5b8b99ca6f4bb6ddec973e59cd4 \ No newline at end of file +fd8d7440277b17f57a6dc0a210f42adf1fa2dcd2 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 48c130935e..651b15f7d2 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3587,6 +3587,14 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); return SQLITE_OK; } + case SQLITE_FCNTL_PRAGMA: { + char **azArg = (char**)pArg; + if( sqlite3_stricmp(azArg[1], "filename")==0 ){ + azArg[0] = sqlite3_mprintf("%s", pFile->zPath); + return SQLITE_OK; + } + break; + } #ifndef NDEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and diff --git a/src/pragma.c b/src/pragma.c index e449baa8fb..079348e11b 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -312,7 +312,7 @@ void sqlite3Pragma( const char *zDb = 0; /* The database name */ Token *pId; /* Pointer to token */ int iDb; /* Database index for */ - const char *aFcntl[3]; /* Argument to SQLITE_FCNTL_PRAGMA */ + char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ @@ -353,13 +353,21 @@ void sqlite3Pragma( ** connection. If it returns SQLITE_OK, then assume that the VFS ** handled the pragma and generate a no-op prepared statement. */ - aFcntl[0] = zLeft; - aFcntl[1] = zRight; - aFcntl[2] = 0; + aFcntl[0] = 0; + aFcntl[1] = zLeft; + aFcntl[2] = zRight; + aFcntl[3] = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ - zLeft = ""; - } + if( aFcntl[0] ){ + int mem = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_String8, 0, mem, 0, aFcntl[0], 0); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "result", SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); + sqlite3_free(aFcntl[0]); + } + }else #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1bd1b08685..f0eb1987ec 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6595,11 +6595,12 @@ int sqlite3_unlock_notify( /* ** CAPI3REF: String Comparison ** -** ^The [sqlite3_strnicmp()] API allows applications and extensions to -** compare the contents of two buffers containing UTF-8 strings in a -** case-independent fashion, using the same definition of case independence -** that SQLite uses internally when comparing identifiers. +** ^The [sqlite3_stricmp()] and [sqlite3_strnicmp()] APIs allow applications +** and extensions to compare the contents of two buffers containing UTF-8 +** strings in a case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. */ +int sqlite3_stricmp(const char *, const char *); int sqlite3_strnicmp(const char *, const char *, int); /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8165d05bff..cfe8fd64fb 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2553,7 +2553,7 @@ int sqlite3CantopenError(int); /* ** Internal function prototypes */ -int sqlite3StrICmp(const char *, const char *); +#define sqlite3StrICmp sqlite3_stricmp int sqlite3Strlen30(const char*); #define sqlite3StrNICmp sqlite3_strnicmp diff --git a/src/test_vfstrace.c b/src/test_vfstrace.c index a1fa6ff442..44a652454d 100644 --- a/src/test_vfstrace.c +++ b/src/test_vfstrace.c @@ -478,7 +478,7 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ case 0xca093fa0: zOp = "DB_UNCHANGED"; break; case SQLITE_FCNTL_PRAGMA: { const char *const* a = (const char*const*)pArg; - sqlite3_snprintf(sizeof(zBuf), zBuf, "PRAGMA,[%s,%s]",a[0],a[1]); + sqlite3_snprintf(sizeof(zBuf), zBuf, "PRAGMA,[%s,%s]",a[1],a[2]); zOp = zBuf; break; } @@ -496,6 +496,10 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ *(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z", pInfo->zVfsName, *(char**)pArg); } + if( op==SQLITE_FCNTL_PRAGMA && rc==SQLITE_OK && *(char**)pArg ){ + vfstrace_printf(pInfo, "%s.xFileControl(%s,%s) returns %s", + pInfo->zVfsName, p->zFNmae, zOp, *(char**)pArg); + } return rc; } diff --git a/src/util.c b/src/util.c index fd3c858ab7..325c75aae1 100644 --- a/src/util.c +++ b/src/util.c @@ -222,7 +222,7 @@ int sqlite3Dequote(char *z){ ** definition of case independence that SQLite uses internally when ** comparing identifiers. */ -int sqlite3StrICmp(const char *zLeft, const char *zRight){ +int sqlite3_stricmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; diff --git a/test/attach.test b/test/attach.test index be5f9881c3..b29ba1709a 100644 --- a/test/attach.test +++ b/test/attach.test @@ -70,6 +70,7 @@ do_test attach-1.3.5 { file tail [sqlite3_db_filename db three] } {} + do_test attach-1.4 { execsql { SELECT * FROM t2; @@ -270,6 +271,16 @@ ifcapable schema_pragmas { } } ;# ifcapable schema_pragmas + +if {[regexp unix [file_control_vfsname db]]} { + do_test attach-1.30.1 { + file tail [db eval {PRAGMA main.filename}] + } {test.db} + do_test attach-1.30.2 { + db eval {PRAGMA temp.filename} + } {} +} + ifcapable {trigger} { # Only do the following tests if triggers are enabled do_test attach-2.1 { execsql { From e4d25e9cfb51dd7d76a7fd95698ce4265ca03b2b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2012 18:21:20 +0000 Subject: [PATCH 26/51] Make benign any malloc failures inside the sqlite3_file_control() call within PRAGMA parsing. Add a couple simple tests for SQLITE_FCNTL_PRAGMA. FossilOrigin-Name: a1f29fa653b788f563967f55c3d20e64cbe3f549 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pragma.c | 2 ++ test/attach.test | 14 ++++++++------ 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 46df865eb6..9a47c0ad55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sSQLITE_FCNTL_PRAGMA\sfile-control\sso\sthat\sit\scan\sreturn\sa\sstring\nvalue. -D 2012-02-22T16:58:36.697 +C Make\sbenign\sany\smalloc\sfailures\sinside\sthe\ssqlite3_file_control()\scall\swithin\nPRAGMA\sparsing.\s\sAdd\sa\scouple\ssimple\stests\sfor\sSQLITE_FCNTL_PRAGMA. +D 2012-02-22T18:21:20.823 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c b30b1c35908346ecc43d8d9d17f2ddf6817f8f60 -F src/pragma.c 6838f56b049840d015fc9b335ab4118479bdaac9 +F src/pragma.c c452bbbd2afc48e80a42adaba559bd63e998ea17 F src/prepare.c ec4989f7f480544bdc4192fe663470d2a2d7d61e F src/printf.c 7ffb4ebb8b341f67e049695ba031da717b3d2699 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -275,7 +275,7 @@ F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 F test/async4.test 1787e3952128aa10238bf39945126de7ca23685a F test/async5.test 0dd8701bd588bf6e70c2557a22ae3f22b2567b4c -F test/attach.test 064911a24b6692759c661b981ed953c5fd9f441a +F test/attach.test 80762134ca101a79dbe3dda5285d7897639057a7 F test/attach2.test e54436ed956d3d88bdee61221da59bf3935a0966 F test/attach3.test d89ccfe4fe6e2b5e368d480fcdfe4b496c54cf4e F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 5be07904baf6e5b8b99ca6f4bb6ddec973e59cd4 -R f8a789f52fcbc32c2be7b7547e20bad5 +P fd8d7440277b17f57a6dc0a210f42adf1fa2dcd2 +R 77f865c04d434abb001a6242927d5d25 U drh -Z 2d0e9a9b9d36c2e692d7eed74ee06028 +Z d262af2f28e1a265375911629e3a952e diff --git a/manifest.uuid b/manifest.uuid index 90e28dbbf0..c29fcb98f3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd8d7440277b17f57a6dc0a210f42adf1fa2dcd2 \ No newline at end of file +a1f29fa653b788f563967f55c3d20e64cbe3f549 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 079348e11b..c5f4b285cd 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -357,7 +357,9 @@ void sqlite3Pragma( aFcntl[1] = zLeft; aFcntl[2] = zRight; aFcntl[3] = 0; + sqlite3BeginBenignMalloc(); rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + sqlite3EndBenignMalloc(); if( rc==SQLITE_OK ){ if( aFcntl[0] ){ int mem = ++pParse->nMem; diff --git a/test/attach.test b/test/attach.test index b29ba1709a..78358980fc 100644 --- a/test/attach.test +++ b/test/attach.test @@ -273,12 +273,14 @@ ifcapable schema_pragmas { if {[regexp unix [file_control_vfsname db]]} { - do_test attach-1.30.1 { - file tail [db eval {PRAGMA main.filename}] - } {test.db} - do_test attach-1.30.2 { - db eval {PRAGMA temp.filename} - } {} + ifcapable debug { + do_test attach-1.30.1 { + file tail [db eval {PRAGMA main.filename}] + } {test.db} + do_test attach-1.30.2 { + db eval {PRAGMA temp.filename} + } {} + } } ifcapable {trigger} { # Only do the following tests if triggers are enabled From eb9a9696f6937c4b9e61050dbb14bbd724a1d047 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2012 19:03:19 +0000 Subject: [PATCH 27/51] Disable the "filename" pragma implemented by os_unix.c except when compiled with SQLITE_TEST. FossilOrigin-Name: 9a1da91850b6074e5f9d18382f517eb868cad63c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9a47c0ad55..3688bf37f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sbenign\sany\smalloc\sfailures\sinside\sthe\ssqlite3_file_control()\scall\swithin\nPRAGMA\sparsing.\s\sAdd\sa\scouple\ssimple\stests\sfor\sSQLITE_FCNTL_PRAGMA. -D 2012-02-22T18:21:20.823 +C Disable\sthe\s"filename"\spragma\simplemented\sby\sos_unix.c\sexcept\swhen\scompiled\nwith\sSQLITE_TEST. +D 2012-02-22T19:03:19.757 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 497e355ab617a875bf45f4d58bacca82952339d5 +F src/os_unix.c 49e10d7d5783df93684b3c232f03394ad7171ee3 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P fd8d7440277b17f57a6dc0a210f42adf1fa2dcd2 -R 77f865c04d434abb001a6242927d5d25 +P a1f29fa653b788f563967f55c3d20e64cbe3f549 +R 69256c5b535e2ca4839b991d3bdc0dd8 U drh -Z d262af2f28e1a265375911629e3a952e +Z 24345afb75f6875260101e2916714082 diff --git a/manifest.uuid b/manifest.uuid index c29fcb98f3..fb3e345492 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1f29fa653b788f563967f55c3d20e64cbe3f549 \ No newline at end of file +9a1da91850b6074e5f9d18382f517eb868cad63c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 651b15f7d2..f747d8a877 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3589,10 +3589,12 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } case SQLITE_FCNTL_PRAGMA: { char **azArg = (char**)pArg; +#ifdef SQLITE_TEST if( sqlite3_stricmp(azArg[1], "filename")==0 ){ azArg[0] = sqlite3_mprintf("%s", pFile->zPath); return SQLITE_OK; } +#endif break; } #ifndef NDEBUG From 92c700dbb7410a3b920974fed37316f30d664e9a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2012 19:56:17 +0000 Subject: [PATCH 28/51] If the SQLITE_FCNTL_PRAGMA file-control returns anything other than SQLTIE_NOTFOUND and SQLITE_OK, then treat the result as an error. FossilOrigin-Name: 5643618108a8aafba67ed4004039b862bb5e5da8 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pragma.c | 11 ++++++++--- src/test_vfs.c | 17 +++++++++++++++++ test/pragma.test | 19 +++++++++++++++++++ 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 3688bf37f2..dddb378ea9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\s"filename"\spragma\simplemented\sby\sos_unix.c\sexcept\swhen\scompiled\nwith\sSQLITE_TEST. -D 2012-02-22T19:03:19.757 +C If\sthe\sSQLITE_FCNTL_PRAGMA\sfile-control\sreturns\sanything\sother\sthan\nSQLTIE_NOTFOUND\sand\sSQLITE_OK,\sthen\streat\sthe\sresult\sas\san\serror. +D 2012-02-22T19:56:17.256 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c b30b1c35908346ecc43d8d9d17f2ddf6817f8f60 -F src/pragma.c c452bbbd2afc48e80a42adaba559bd63e998ea17 +F src/pragma.c a6b1b209a143fd235fe61cad4bf4e0d1f765f5ce F src/prepare.c ec4989f7f480544bdc4192fe663470d2a2d7d61e F src/printf.c 7ffb4ebb8b341f67e049695ba031da717b3d2699 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -230,7 +230,7 @@ F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 -F src/test_vfs.c 07157a0bbfe161cb5e32cad2079abd26cd611c4b +F src/test_vfs.c 37d523ce239e853a302933327ccbc8c73c4572c1 F src/test_vfstrace.c f20a32a5fd46feb59a93d2cf650f1ff92da3872b F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 @@ -635,7 +635,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/permutations.test fa6f0e5f13fe0b1d3f7a7613179b7f7b20028184 -F test/pragma.test 7fa35e53085812dac94c2bfcbb02c2a4ad35df5e +F test/pragma.test 01abcc2e43d080db8df0f9517233937c20892bd1 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P a1f29fa653b788f563967f55c3d20e64cbe3f549 -R 69256c5b535e2ca4839b991d3bdc0dd8 +P 9a1da91850b6074e5f9d18382f517eb868cad63c +R 38bcf4842421c21898ae536d39030b7a U drh -Z 24345afb75f6875260101e2916714082 +Z 347350cb546f31164cbf5b1b4da5c224 diff --git a/manifest.uuid b/manifest.uuid index fb3e345492..fb958af863 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a1da91850b6074e5f9d18382f517eb868cad63c \ No newline at end of file +5643618108a8aafba67ed4004039b862bb5e5da8 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index c5f4b285cd..7330e39d14 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -357,9 +357,7 @@ void sqlite3Pragma( aFcntl[1] = zLeft; aFcntl[2] = zRight; aFcntl[3] = 0; - sqlite3BeginBenignMalloc(); rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); - sqlite3EndBenignMalloc(); if( rc==SQLITE_OK ){ if( aFcntl[0] ){ int mem = ++pParse->nMem; @@ -369,7 +367,14 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); sqlite3_free(aFcntl[0]); } - }else + }else if( rc!=SQLITE_NOTFOUND ){ + if( aFcntl[0] ){ + sqlite3ErrorMsg(pParse, "%s", aFcntl[0]); + sqlite3_free(aFcntl[0]); + } + pParse->nErr++; + pParse->rc = rc; + } #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) diff --git a/src/test_vfs.c b/src/test_vfs.c index a79407b57f..c18d0f6dae 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -480,6 +480,23 @@ static int tvfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){ */ static int tvfsFileControl(sqlite3_file *pFile, int op, void *pArg){ TestvfsFd *p = tvfsGetFd(pFile); + if( op==SQLITE_FCNTL_PRAGMA ){ + char **argv = (char**)pArg; + if( sqlite3_stricmp(argv[1],"error")==0 ){ + int rc = SQLITE_ERROR; + if( argv[2] ){ + const char *z = argv[2]; + int x = atoi(z); + if( x ){ + rc = x; + while( sqlite3Isdigit(z[0]) ){ z++; } + while( sqlite3Isspace(z[0]) ){ z++; } + } + if( z[0] ) argv[0] = sqlite3_mprintf("%s", z); + } + return rc; + } + } return sqlite3OsFileControl(p->pReal, op, pArg); } diff --git a/test/pragma.test b/test/pragma.test index cbe35ea806..fb1034f10a 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1489,4 +1489,23 @@ foreach {temp_setting val} { } $val } +# The SQLITE_FCNTL_PRAGMA logic, with error handling. +# +db close +testvfs tvfs +sqlite3 db test.db -vfs tvfs +do_test pragma-19.1 { + catchsql {PRAGMA error} +} {1 {SQL logic error or missing database}} +do_test pragma-19.2 { + catchsql {PRAGMA error='This is the error message'} +} {1 {This is the error message}} +do_test pragma-19.3 { + catchsql {PRAGMA error='7 This is the error message'} +} {1 {This is the error message}} +do_test pragma-19.4 { + catchsql {PRAGMA error=7} +} {1 {out of memory}} + + finish_test From c8517f614dbfc2ffecd4c72031c24283c8dfa69b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Feb 2012 20:08:49 +0000 Subject: [PATCH 29/51] Move test logic for SQLITE_FCNTL_PRAGMA out of os_unix.c and into test_vfs.c. FossilOrigin-Name: c81fc40b2b584820ac7d1c3848ebeb7225d4eeeb --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 10 ---------- src/test_vfs.c | 4 ++++ test/attach.test | 11 ----------- test/pragma.test | 3 +++ 6 files changed, 17 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index dddb378ea9..8c2bdd225f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\sSQLITE_FCNTL_PRAGMA\sfile-control\sreturns\sanything\sother\sthan\nSQLTIE_NOTFOUND\sand\sSQLITE_OK,\sthen\streat\sthe\sresult\sas\san\serror. -D 2012-02-22T19:56:17.256 +C Move\stest\slogic\sfor\sSQLITE_FCNTL_PRAGMA\sout\sof\sos_unix.c\sand\sinto\stest_vfs.c. +D 2012-02-22T20:08:49.412 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 59beba555b65a450bd1d804220532971d4299f60 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 49e10d7d5783df93684b3c232f03394ad7171ee3 +F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a @@ -230,7 +230,7 @@ F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 -F src/test_vfs.c 37d523ce239e853a302933327ccbc8c73c4572c1 +F src/test_vfs.c 73f46bd9b5183ebcb77da22773886b81157cdc3d F src/test_vfstrace.c f20a32a5fd46feb59a93d2cf650f1ff92da3872b F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 @@ -275,7 +275,7 @@ F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 F test/async4.test 1787e3952128aa10238bf39945126de7ca23685a F test/async5.test 0dd8701bd588bf6e70c2557a22ae3f22b2567b4c -F test/attach.test 80762134ca101a79dbe3dda5285d7897639057a7 +F test/attach.test 8511625fa8c7dabbcbc48fc491c0b07bae65fb0e F test/attach2.test e54436ed956d3d88bdee61221da59bf3935a0966 F test/attach3.test d89ccfe4fe6e2b5e368d480fcdfe4b496c54cf4e F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c @@ -635,7 +635,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/permutations.test fa6f0e5f13fe0b1d3f7a7613179b7f7b20028184 -F test/pragma.test 01abcc2e43d080db8df0f9517233937c20892bd1 +F test/pragma.test f6111ded4d56b79436a60a757d62f3c96a9cf3f5 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 9a1da91850b6074e5f9d18382f517eb868cad63c -R 38bcf4842421c21898ae536d39030b7a +P 5643618108a8aafba67ed4004039b862bb5e5da8 +R 6e8e292599506d27fedbb78f0e61a658 U drh -Z 347350cb546f31164cbf5b1b4da5c224 +Z b333c9e71d55e82ea7cb2edfc422542b diff --git a/manifest.uuid b/manifest.uuid index fb958af863..adc52f659c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5643618108a8aafba67ed4004039b862bb5e5da8 \ No newline at end of file +c81fc40b2b584820ac7d1c3848ebeb7225d4eeeb \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f747d8a877..48c130935e 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3587,16 +3587,6 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); return SQLITE_OK; } - case SQLITE_FCNTL_PRAGMA: { - char **azArg = (char**)pArg; -#ifdef SQLITE_TEST - if( sqlite3_stricmp(azArg[1], "filename")==0 ){ - azArg[0] = sqlite3_mprintf("%s", pFile->zPath); - return SQLITE_OK; - } -#endif - break; - } #ifndef NDEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and diff --git a/src/test_vfs.c b/src/test_vfs.c index c18d0f6dae..bab67798cc 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -496,6 +496,10 @@ static int tvfsFileControl(sqlite3_file *pFile, int op, void *pArg){ } return rc; } + if( sqlite3_stricmp(argv[1], "filename")==0 ){ + argv[0] = sqlite3_mprintf("%s", p->zFilename); + return SQLITE_OK; + } } return sqlite3OsFileControl(p->pReal, op, pArg); } diff --git a/test/attach.test b/test/attach.test index 78358980fc..008eee65f6 100644 --- a/test/attach.test +++ b/test/attach.test @@ -272,17 +272,6 @@ ifcapable schema_pragmas { } ;# ifcapable schema_pragmas -if {[regexp unix [file_control_vfsname db]]} { - ifcapable debug { - do_test attach-1.30.1 { - file tail [db eval {PRAGMA main.filename}] - } {test.db} - do_test attach-1.30.2 { - db eval {PRAGMA temp.filename} - } {} - } -} - ifcapable {trigger} { # Only do the following tests if triggers are enabled do_test attach-2.1 { execsql { diff --git a/test/pragma.test b/test/pragma.test index fb1034f10a..00be9b8f6a 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1506,6 +1506,9 @@ do_test pragma-19.3 { do_test pragma-19.4 { catchsql {PRAGMA error=7} } {1 {out of memory}} +do_test pragma-19.5 { + file tail [execsql {PRAGMA filename}] +} {test.db} finish_test From b3bdc72deda391ba7867ba14afa18fbd433cb12e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 23 Feb 2012 15:35:49 +0000 Subject: [PATCH 30/51] Add the sqlite3PagerWalFramesize() function to pager.c. This is used by zipvfs. FossilOrigin-Name: 786e24dc56504d1958db0618e645610f2600a087 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/pager.c | 12 ++++++++++++ src/pager.h | 2 ++ src/wal.c | 11 +++++++++++ src/wal.h | 6 ++++++ 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c288bd5a5e..3b970f297d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expose\sthe\ssqlite3_stricmp()\sinterface.\s\sAdd\sthe\sSQLITE_FCNTL_PRAGMA\sfile\ncontrol. -D 2012-02-23T14:28:46.056 +C Add\sthe\ssqlite3PagerWalFramesize()\sfunction\sto\spager.c.\sThis\sis\sused\sby\szipvfs. +D 2012-02-23T15:35:49.591 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,8 +168,8 @@ F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 -F src/pager.c 2d892f7b901a8867a33bc21742086165a3a99af8 -F src/pager.h a435da8421dc7844b7f9c7f37b636c160c50208a +F src/pager.c af82f106e5b22e90cdbccbccdabe14bfa00ced5e +F src/pager.h 0ccc8260fadff8e0ce5193392faea6954178bf3d F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c @@ -250,8 +250,8 @@ F src/vdbemem.c fb0ac964ccbcd94f595eb993c05bfd9c52468a4a F src/vdbesort.c b25814d385895544ebc8118245c8311ded7f81c9 F src/vdbetrace.c d6e50e04e1ec498150e519058f617d91b8f5c843 F src/vtab.c ab90fb600a3f5e4b7c48d22a4cdb2d6b23239847 -F src/wal.c 5f7bcc0610af759953defd769eacebfd98a22bc8 -F src/wal.h eaa00b9a403ddda2b56d01b7afc19ef600f9363f +F src/wal.c 176b50fc9e30ef76d3d3c9ba99a40cea6f23eace +F src/wal.h 9a300673e86f04eeb7442348d3557eecc5a9509a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c af623942514571895818b9b7ae11db95ae3b3d88 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P f8ac826f95ae6b7b981e4765d2e65b6839d0eff9 c81fc40b2b584820ac7d1c3848ebeb7225d4eeeb -R c32d45c36d618b8574dcc1d6ea477e07 -U drh -Z 6215aa1a94e7aa4510913be189893b44 +P c539cf21ef8f4eb4782790aa97a67d10ae9ffd64 +R 9cc56f07006d3b03439dff4deb3c69b5 +U dan +Z 4dc0dc11213b3ebf076551ec72da6775 diff --git a/manifest.uuid b/manifest.uuid index c31cd074b7..d8472242ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c539cf21ef8f4eb4782790aa97a67d10ae9ffd64 \ No newline at end of file +786e24dc56504d1958db0618e645610f2600a087 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index be360448f0..d293472957 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6871,6 +6871,18 @@ int sqlite3PagerCloseWal(Pager *pPager){ return rc; } +/* +** A read-lock must be held on the pager when this function is called. If +** the pager is in WAL mode and the WAL file currently contains one or more +** frames, return the size in bytes of the page images stored within the +** WAL frames. Otherwise, if this is not a WAL database or the WAL file +** is empty, return 0. +*/ +int sqlite3PagerWalFramesize(Pager *pPager){ + assert( pPager->eState==PAGER_SHARED ); + return sqlite3WalFramesize(pPager->pWal); +} + #ifdef SQLITE_HAS_CODEC /* ** This function is called by the wal module when writing page content diff --git a/src/pager.h b/src/pager.h index e5088bdabc..240aeb4b5d 100644 --- a/src/pager.h +++ b/src/pager.h @@ -144,6 +144,8 @@ int sqlite3PagerWalCallback(Pager *pPager); int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); int sqlite3PagerCloseWal(Pager *pPager); +int sqlite3PagerWalFramesize(Pager *pPager); + /* Functions used to query pager state and configuration. */ u8 sqlite3PagerIsreadonly(Pager*); int sqlite3PagerRefcount(Pager*); diff --git a/src/wal.c b/src/wal.c index db9ce5186f..315fc0bc26 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3068,4 +3068,15 @@ int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } +/* +** If the argument is not NULL, it points to a Wal object that holds a +** read-lock. This function returns the database page-size if it is known, +** or zero if it is not (or if pWal is NULL). +*/ +int sqlite3WalFramesize(Wal *pWal){ + int nRet = 0; + assert( pWal==0 || pWal->readLock>=0 ); + return (pWal ? pWal->szPage : 0); +} + #endif /* #ifndef SQLITE_OMIT_WAL */ diff --git a/src/wal.h b/src/wal.h index fac762e870..437acd1df3 100644 --- a/src/wal.h +++ b/src/wal.h @@ -43,6 +43,7 @@ # define sqlite3WalCallback(z) 0 # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 +# define sqlite3WalFramesize(z) 0 #else #define WAL_SAVEPOINT_NDATA 4 @@ -124,5 +125,10 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); +/* If the WAL file is not empty, return the number of bytes of content +** stored in each frame (i.e. the db page-size when the WAL was created). +*/ +int sqlite3WalFramesize(Wal *pWal); + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* _WAL_H_ */ From bf24261b7beda34593596dffa77108e176bbce1c Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 23 Feb 2012 16:45:42 +0000 Subject: [PATCH 31/51] Fix a typo made inside an assert() statement in the previous commit. FossilOrigin-Name: f0640615f25ebe8bf2eed0e1a0f2e309471889fe --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 2 +- src/wal.c | 1 - 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3b970f297d..7316e3191d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3PagerWalFramesize()\sfunction\sto\spager.c.\sThis\sis\sused\sby\szipvfs. -D 2012-02-23T15:35:49.591 +C Fix\sa\stypo\smade\sinside\san\sassert()\sstatement\sin\sthe\sprevious\scommit. +D 2012-02-23T16:45:42.651 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 -F src/pager.c af82f106e5b22e90cdbccbccdabe14bfa00ced5e +F src/pager.c 25733ab94de3c058b98b7481ed0d7163cd9471cf F src/pager.h 0ccc8260fadff8e0ce5193392faea6954178bf3d F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -250,7 +250,7 @@ F src/vdbemem.c fb0ac964ccbcd94f595eb993c05bfd9c52468a4a F src/vdbesort.c b25814d385895544ebc8118245c8311ded7f81c9 F src/vdbetrace.c d6e50e04e1ec498150e519058f617d91b8f5c843 F src/vtab.c ab90fb600a3f5e4b7c48d22a4cdb2d6b23239847 -F src/wal.c 176b50fc9e30ef76d3d3c9ba99a40cea6f23eace +F src/wal.c ef05bdbff7ad30eb950222b2783dac21f7b8acb4 F src/wal.h 9a300673e86f04eeb7442348d3557eecc5a9509a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c af623942514571895818b9b7ae11db95ae3b3d88 @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P c539cf21ef8f4eb4782790aa97a67d10ae9ffd64 -R 9cc56f07006d3b03439dff4deb3c69b5 +P 786e24dc56504d1958db0618e645610f2600a087 +R 62ee79e3c7935d38f24dc9d649f992a8 U dan -Z 4dc0dc11213b3ebf076551ec72da6775 +Z 453c5f133ed9456db2536b040019e9b7 diff --git a/manifest.uuid b/manifest.uuid index d8472242ac..9847f54855 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -786e24dc56504d1958db0618e645610f2600a087 \ No newline at end of file +f0640615f25ebe8bf2eed0e1a0f2e309471889fe \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d293472957..2cf4dd0dd9 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6879,7 +6879,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ ** is empty, return 0. */ int sqlite3PagerWalFramesize(Pager *pPager){ - assert( pPager->eState==PAGER_SHARED ); + assert( pPager->eState==PAGER_READER ); return sqlite3WalFramesize(pPager->pWal); } diff --git a/src/wal.c b/src/wal.c index 315fc0bc26..48fcba100b 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3074,7 +3074,6 @@ int sqlite3WalHeapMemory(Wal *pWal){ ** or zero if it is not (or if pWal is NULL). */ int sqlite3WalFramesize(Wal *pWal){ - int nRet = 0; assert( pWal==0 || pWal->readLock>=0 ); return (pWal ? pWal->szPage : 0); } From 63296052a18ec880aab46c1f9a140f9ab0a6b271 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Feb 2012 17:35:28 +0000 Subject: [PATCH 32/51] Fix an obscure bug that causes a crash when parsing certain invalid SQL statements. FossilOrigin-Name: c8c7846fb990199dcc0774a69e8257510bc36f2d --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 2 +- test/select1.test | 6 ++++++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 7316e3191d..99e12674d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\smade\sinside\san\sassert()\sstatement\sin\sthe\sprevious\scommit. -D 2012-02-23T16:45:42.651 +C Fix\san\sobscure\sbug\sthat\scauses\sa\scrash\swhen\sparsing\scertain\sinvalid\sSQL\nstatements. +D 2012-02-23T17:35:28.314 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/printf.c 7ffb4ebb8b341f67e049695ba031da717b3d2699 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 232283a2e60d91cbd9a5ddf2f6f7ecf53d590075 +F src/select.c 44ccdcb5d2a1c48622c179b2d72167b716388581 F src/shell.c aa28f117033ba3e44b5eaaf2ad572222bcdfd66e F src/sqlite.h.in f46e368d1a28b09d876e35444785674d170f2d62 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 @@ -666,7 +666,7 @@ F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5 F test/securedel.test 87a2561151af1f1e349071a89fdd77059f50113c -F test/select1.test f67ca2dfc05df41c7b86eb32ca409b427a5f43b0 +F test/select1.test deba017eed9daa5af33de868676c997e7eebb931 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054 F test/select4.test 44aa6e7110592e18110b0b9cf5c024d37d23be17 @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 786e24dc56504d1958db0618e645610f2600a087 -R 62ee79e3c7935d38f24dc9d649f992a8 -U dan -Z 453c5f133ed9456db2536b040019e9b7 +P f0640615f25ebe8bf2eed0e1a0f2e309471889fe +R 386c50c5bbaa55f00d88b4b7b8d3f590 +U drh +Z 9c1aa4f559c8a629e9a1fae5ce79c6c5 diff --git a/manifest.uuid b/manifest.uuid index 9847f54855..4299d78489 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f0640615f25ebe8bf2eed0e1a0f2e309471889fe \ No newline at end of file +c8c7846fb990199dcc0774a69e8257510bc36f2d \ No newline at end of file diff --git a/src/select.c b/src/select.c index bbdc4e3c55..3efe014d8b 100644 --- a/src/select.c +++ b/src/select.c @@ -1258,7 +1258,7 @@ static int selectColumnsFromExprList( char *zName; /* Column name */ int nName; /* Size of name in zName[] */ - *pnCol = nCol = pEList->nExpr; + *pnCol = nCol = pEList ? pEList->nExpr : 0; aCol = *paCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); if( aCol==0 ) return SQLITE_NOMEM; for(i=0, pCol=aCol; i Date: Fri, 24 Feb 2012 00:02:28 +0000 Subject: [PATCH 33/51] A simple change that allows the SQLITE_32BIT_ROWID compile-time option to work better. FossilOrigin-Name: f9df4063fbd100ce306a89fc649c1860fd801f31 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 99e12674d4..ff6ebe0f79 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sobscure\sbug\sthat\scauses\sa\scrash\swhen\sparsing\scertain\sinvalid\sSQL\nstatements. -D 2012-02-23T17:35:28.314 +C A\ssimple\schange\sthat\sallows\sthe\sSQLITE_32BIT_ROWID\scompile-time\soption\sto\nwork\sbetter. +D 2012-02-24T00:02:28.766 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -240,7 +240,7 @@ F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 906731099c4397bf8adf3fa90a833355e7472af0 F src/vacuum.c 0c0ba2242355c6048d65e2b333abe0f7c06348fa -F src/vdbe.c 21c17c43e579789959efea50f100de60efbb955c +F src/vdbe.c 32720e873ed0a23e6ee928b676cd995864b984d6 F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 F src/vdbeapi.c 3662b6a468a2a4605a15dfab313baa6dff81ad91 @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P f0640615f25ebe8bf2eed0e1a0f2e309471889fe -R 386c50c5bbaa55f00d88b4b7b8d3f590 +P c8c7846fb990199dcc0774a69e8257510bc36f2d +R c536366575fdc7c1acdfd3976080e3e1 U drh -Z 9c1aa4f559c8a629e9a1fae5ce79c6c5 +Z e5f81487e506b4905e600ae695ff035e diff --git a/manifest.uuid b/manifest.uuid index 4299d78489..479220e768 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8c7846fb990199dcc0774a69e8257510bc36f2d \ No newline at end of file +f9df4063fbd100ce306a89fc649c1860fd801f31 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 81cc0c3d4c..749830bc0d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3844,7 +3844,7 @@ case OP_NewRowid: { /* out2-prerelease */ assert( sqlite3BtreeCursorIsValid(pC->pCursor) ); rc = sqlite3BtreeKeySize(pC->pCursor, &v); assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ - if( v==MAX_ROWID ){ + if( v>=MAX_ROWID ){ pC->useRandomRowid = 1; }else{ v++; /* IMP: R-29538-34987 */ From 8d93684874cd3cde65f0c433e32d840de09af9f4 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Feb 2012 00:03:12 +0000 Subject: [PATCH 34/51] If the SQLITE_FCNTL_PRAGMA file control returns something other than SQLITE_NOTFOUND, then skip the rest of Pragma code generation. FossilOrigin-Name: 9e6d340f557c2f2c8a07300ac60357af9906a7fa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ff6ebe0f79..ddf61903cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\ssimple\schange\sthat\sallows\sthe\sSQLITE_32BIT_ROWID\scompile-time\soption\sto\nwork\sbetter. -D 2012-02-24T00:02:28.766 +C If\sthe\sSQLITE_FCNTL_PRAGMA\sfile\scontrol\sreturns\ssomething\sother\sthan\nSQLITE_NOTFOUND,\sthen\sskip\sthe\srest\sof\sPragma\scode\sgeneration. +D 2012-02-24T00:03:12.589 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c b30b1c35908346ecc43d8d9d17f2ddf6817f8f60 -F src/pragma.c a6b1b209a143fd235fe61cad4bf4e0d1f765f5ce +F src/pragma.c 11f5fff828e00ece318eefc8c8808b4fc9835739 F src/prepare.c ec4989f7f480544bdc4192fe663470d2a2d7d61e F src/printf.c 7ffb4ebb8b341f67e049695ba031da717b3d2699 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P c8c7846fb990199dcc0774a69e8257510bc36f2d -R c536366575fdc7c1acdfd3976080e3e1 +P f9df4063fbd100ce306a89fc649c1860fd801f31 +R 918c6dd4ec2727a8cb49f3dbede7e930 U drh -Z e5f81487e506b4905e600ae695ff035e +Z a93fbf9532fae037d0b7a67217939ce5 diff --git a/manifest.uuid b/manifest.uuid index 479220e768..40120e066a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9df4063fbd100ce306a89fc649c1860fd801f31 \ No newline at end of file +9e6d340f557c2f2c8a07300ac60357af9906a7fa \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 7330e39d14..bc1469b798 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -374,7 +374,7 @@ void sqlite3Pragma( } pParse->nErr++; pParse->rc = rc; - } + }else #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) From 70708600b533734a2ef7e2312674692f513a640c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Feb 2012 14:33:28 +0000 Subject: [PATCH 35/51] Add #ifdef SQLITE_ENABLE_ZIPVFS around those bits of code that are only used by ZIPVFS. FossilOrigin-Name: 3130275c64a60c5cfacbd11123305e6d99ea866d --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/pager.c | 2 ++ src/pager.h | 5 +++-- src/pragma.c | 3 +-- src/wal.c | 2 ++ src/wal.h | 2 ++ 7 files changed, 21 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index ddf61903cc..f6b0508b64 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\sSQLITE_FCNTL_PRAGMA\sfile\scontrol\sreturns\ssomething\sother\sthan\nSQLITE_NOTFOUND,\sthen\sskip\sthe\srest\sof\sPragma\scode\sgeneration. -D 2012-02-24T00:03:12.589 +C Add\s#ifdef\sSQLITE_ENABLE_ZIPVFS\saround\sthose\sbits\sof\scode\sthat\sare\sonly\nused\sby\sZIPVFS. +D 2012-02-24T14:33:28.413 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,13 +168,13 @@ F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 0e3d2942d228d0366fb80a3640f35caf413b66d1 F src/os_win.c 5ac061ae1326a71500cee578ed0fd9113b4f6a37 -F src/pager.c 25733ab94de3c058b98b7481ed0d7163cd9471cf -F src/pager.h 0ccc8260fadff8e0ce5193392faea6954178bf3d +F src/pager.c 3955b62cdb5bb64559607cb474dd12a6c8e1d4a5 +F src/pager.h ef1eaf8593e78f73885c1dfac27ad83bee23bdc5 F src/parse.y 1ddd71ae55f4b7cbb2672526ea4de023de0f519e F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c b30b1c35908346ecc43d8d9d17f2ddf6817f8f60 -F src/pragma.c 11f5fff828e00ece318eefc8c8808b4fc9835739 +F src/pragma.c e708b3bb5704605816f617e0b1d63a5488060715 F src/prepare.c ec4989f7f480544bdc4192fe663470d2a2d7d61e F src/printf.c 7ffb4ebb8b341f67e049695ba031da717b3d2699 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -250,8 +250,8 @@ F src/vdbemem.c fb0ac964ccbcd94f595eb993c05bfd9c52468a4a F src/vdbesort.c b25814d385895544ebc8118245c8311ded7f81c9 F src/vdbetrace.c d6e50e04e1ec498150e519058f617d91b8f5c843 F src/vtab.c ab90fb600a3f5e4b7c48d22a4cdb2d6b23239847 -F src/wal.c ef05bdbff7ad30eb950222b2783dac21f7b8acb4 -F src/wal.h 9a300673e86f04eeb7442348d3557eecc5a9509a +F src/wal.c c164c39e8625b2b6e7d7b2aab5e7296ebd7e8190 +F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c af623942514571895818b9b7ae11db95ae3b3d88 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P f9df4063fbd100ce306a89fc649c1860fd801f31 -R 918c6dd4ec2727a8cb49f3dbede7e930 +P 9e6d340f557c2f2c8a07300ac60357af9906a7fa +R ea57edcd6f93a841cf2e153be1c9f0e7 U drh -Z a93fbf9532fae037d0b7a67217939ce5 +Z 92fb8a3e30d52bd11c67570a5d62f0a2 diff --git a/manifest.uuid b/manifest.uuid index 40120e066a..3f96ea1fda 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9e6d340f557c2f2c8a07300ac60357af9906a7fa \ No newline at end of file +3130275c64a60c5cfacbd11123305e6d99ea866d \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 2cf4dd0dd9..66252aa0f7 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6871,6 +6871,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ return rc; } +#ifdef SQLITE_ENABLE_ZIPVFS /* ** A read-lock must be held on the pager when this function is called. If ** the pager is in WAL mode and the WAL file currently contains one or more @@ -6882,6 +6883,7 @@ int sqlite3PagerWalFramesize(Pager *pPager){ assert( pPager->eState==PAGER_READER ); return sqlite3WalFramesize(pPager->pWal); } +#endif #ifdef SQLITE_HAS_CODEC /* diff --git a/src/pager.h b/src/pager.h index 240aeb4b5d..eca8a2f077 100644 --- a/src/pager.h +++ b/src/pager.h @@ -143,8 +143,9 @@ int sqlite3PagerWalSupported(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); int sqlite3PagerCloseWal(Pager *pPager); - -int sqlite3PagerWalFramesize(Pager *pPager); +#ifdef SQLITE_ENABLE_ZIPVFS + int sqlite3PagerWalFramesize(Pager *pPager); +#endif /* Functions used to query pager state and configuration. */ u8 sqlite3PagerIsreadonly(Pager*); diff --git a/src/pragma.c b/src/pragma.c index bc1469b798..2db0b61508 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -663,7 +663,7 @@ void sqlite3Pragma( ** creates the database file. It is important that it is created ** as an auto-vacuum capable db. */ - int rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); + rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ /* When setting the auto_vacuum mode to either "full" or ** "incremental", write the value of meta[6] in the database @@ -781,7 +781,6 @@ void sqlite3Pragma( }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ - int rc; int res; rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ diff --git a/src/wal.c b/src/wal.c index 48fcba100b..97d2185ea2 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3068,6 +3068,7 @@ int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); } +#ifdef SQLITE_ENABLE_ZIPVFS /* ** If the argument is not NULL, it points to a Wal object that holds a ** read-lock. This function returns the database page-size if it is known, @@ -3077,5 +3078,6 @@ int sqlite3WalFramesize(Wal *pWal){ assert( pWal==0 || pWal->readLock>=0 ); return (pWal ? pWal->szPage : 0); } +#endif #endif /* #ifndef SQLITE_OMIT_WAL */ diff --git a/src/wal.h b/src/wal.h index 437acd1df3..a848de1527 100644 --- a/src/wal.h +++ b/src/wal.h @@ -125,10 +125,12 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op); */ int sqlite3WalHeapMemory(Wal *pWal); +#ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). */ int sqlite3WalFramesize(Wal *pWal); +#endif #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* _WAL_H_ */ From 1a3dd3d445313fb5f1ea8baf459aa80cda6e6fb9 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Feb 2012 15:57:13 +0000 Subject: [PATCH 36/51] Fix the multiplexor so that it works with WAL mode and 8+3 filenames. FossilOrigin-Name: a9fcb46bc08c29aebaf3bc999f45846c3c320f68 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_multiplex.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f6b0508b64..2b9beab832 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s#ifdef\sSQLITE_ENABLE_ZIPVFS\saround\sthose\sbits\sof\scode\sthat\sare\sonly\nused\sby\sZIPVFS. -D 2012-02-24T14:33:28.413 +C Fix\sthe\smultiplexor\sso\sthat\sit\sworks\swith\sWAL\smode\sand\s8+3\sfilenames. +D 2012-02-24T15:57:13.518 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c a6a6baf343f79b942331f13378d045e7e270ae64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c cfe25d74333892ababde61196821a889b4756dee -F src/test_multiplex.c afab2c9d292677ab31e0dd4b3dde2420fb655c5f +F src/test_multiplex.c 31fb992ab388a23813c0cb003dc3c0463aa149ac F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 9e6d340f557c2f2c8a07300ac60357af9906a7fa -R ea57edcd6f93a841cf2e153be1c9f0e7 +P 3130275c64a60c5cfacbd11123305e6d99ea866d +R a63fc7ff56ad9219f116d84627834be8 U drh -Z 92fb8a3e30d52bd11c67570a5d62f0a2 +Z 13fd34b3ced0bed8422321ec5813b610 diff --git a/manifest.uuid b/manifest.uuid index 3f96ea1fda..c4a60f684a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3130275c64a60c5cfacbd11123305e6d99ea866d \ No newline at end of file +a9fcb46bc08c29aebaf3bc999f45846c3c320f68 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index af41c0b61c..ecef3f08c7 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -256,7 +256,7 @@ static void multiplexFilename( int i; for(i=n-1; i>0 && i>=n-4 && zOut[i]!='.'; i--){} if( i>=n-4 ) n = i+1; - if( flags & SQLITE_OPEN_MAIN_JOURNAL ){ + if( flags & (SQLITE_OPEN_MAIN_JOURNAL|SQLITE_OPEN_WAL) ){ /* The extensions on overflow files for main databases are 001, 002, ** 003 and so forth. To avoid name collisions, add 400 to the ** extensions of journal files so that they are 401, 402, 403, .... From 5fb96aaf74921ae6cc0a981d736f0301c75940c7 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Feb 2012 16:13:55 +0000 Subject: [PATCH 37/51] Separate multiplexor overflow pages for -wal and -journal files into separate namespaces when in 8+3 filename mode. FossilOrigin-Name: d6850667ea61172ea0ebea4a22b6b306f74775fa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_multiplex.c | 19 ++++++++++++++----- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 2b9beab832..70063c0c1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\smultiplexor\sso\sthat\sit\sworks\swith\sWAL\smode\sand\s8+3\sfilenames. -D 2012-02-24T15:57:13.518 +C Separate\smultiplexor\soverflow\spages\sfor\s-wal\sand\s-journal\sfiles\sinto\nseparate\snamespaces\swhen\sin\s8+3\sfilename\smode. +D 2012-02-24T16:13:55.243 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c a6a6baf343f79b942331f13378d045e7e270ae64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c cfe25d74333892ababde61196821a889b4756dee -F src/test_multiplex.c 31fb992ab388a23813c0cb003dc3c0463aa149ac +F src/test_multiplex.c 87fcd0b643487e045ca31e61ac9e439eb2fdf376 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 3130275c64a60c5cfacbd11123305e6d99ea866d -R a63fc7ff56ad9219f116d84627834be8 +P a9fcb46bc08c29aebaf3bc999f45846c3c320f68 +R 711b08b1e1970a07421b68efd25cb3be U drh -Z 13fd34b3ced0bed8422321ec5813b610 +Z 36a285e495a18fe31688fe55471fb8b5 diff --git a/manifest.uuid b/manifest.uuid index c4a60f684a..99d0087222 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9fcb46bc08c29aebaf3bc999f45846c3c320f68 \ No newline at end of file +d6850667ea61172ea0ebea4a22b6b306f74775fa \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index ecef3f08c7..bd1e168062 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -81,8 +81,12 @@ #define sqlite3_mutex_notheld(X) ((void)(X),1) #endif /* SQLITE_THREADSAFE==0 */ +/* Maximum chunk number */ +#define MX_CHUNK_NUMBER 299 + /* First chunk for rollback journal files */ #define SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET 400 +#define SQLITE_MULTIPLEX_WAL_8_3_OFFSET 700 /************************ Shim Definitions ******************************/ @@ -251,17 +255,22 @@ static void multiplexFilename( ){ int n = nBase; memcpy(zOut, zBase, n+1); - if( iChunk!=0 && iChunk!=SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET ){ + if( iChunk!=0 && iChunk<=MX_CHUNK_NUMBER ){ #ifdef SQLITE_ENABLE_8_3_NAMES int i; for(i=n-1; i>0 && i>=n-4 && zOut[i]!='.'; i--){} if( i>=n-4 ) n = i+1; - if( flags & (SQLITE_OPEN_MAIN_JOURNAL|SQLITE_OPEN_WAL) ){ + if( flags & SQLITE_OPEN_MAIN_JOURNAL ){ /* The extensions on overflow files for main databases are 001, 002, - ** 003 and so forth. To avoid name collisions, add 400 to the - ** extensions of journal files so that they are 401, 402, 403, .... - */ + ** 003 and so forth. To avoid name collisions, add 400 to the + ** extensions of journal files so that they are 401, 402, 403, .... + */ iChunk += SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET; + }else if( flags & SQLITE_OPEN_WAL ){ + /* To avoid name collisions, add 700 to the + ** extensions of WAL files so that they are 701, 702, 703, .... + */ + iChunk += SQLITE_MULTIPLEX_WAL_8_3_OFFSET; } #endif sqlite3_snprintf(4,&zOut[n],"%03d",iChunk); From c611ee94bb629d44c2886b4aa2acf3c1d7643186 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Feb 2012 14:28:50 +0000 Subject: [PATCH 38/51] Fix the xDelete implementation of the multiplexor VFS so that it correctly deletes overflow WAL files. FossilOrigin-Name: e44e26771e5aa48f3bfa14d6db3e669b5b2a1e73 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_multiplex.c | 9 +++++++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 70063c0c1a..0b41ca5008 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Separate\smultiplexor\soverflow\spages\sfor\s-wal\sand\s-journal\sfiles\sinto\nseparate\snamespaces\swhen\sin\s8+3\sfilename\smode. -D 2012-02-24T16:13:55.243 +C Fix\sthe\sxDelete\simplementation\sof\sthe\smultiplexor\sVFS\sso\sthat\sit\scorrectly\ndeletes\soverflow\sWAL\sfiles. +D 2012-02-27T14:28:50.338 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c a6a6baf343f79b942331f13378d045e7e270ae64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c cfe25d74333892ababde61196821a889b4756dee -F src/test_multiplex.c 87fcd0b643487e045ca31e61ac9e439eb2fdf376 +F src/test_multiplex.c 0b3111a8c1b11963b2ca17a4b47219a5942bc9c0 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P a9fcb46bc08c29aebaf3bc999f45846c3c320f68 -R 711b08b1e1970a07421b68efd25cb3be +P d6850667ea61172ea0ebea4a22b6b306f74775fa +R ea71d798aaf411fbd421b77e33a3053e U drh -Z 36a285e495a18fe31688fe55471fb8b5 +Z 3207ff66d565aff181a5c8c10cee009f diff --git a/manifest.uuid b/manifest.uuid index 99d0087222..e813cbd424 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6850667ea61172ea0ebea4a22b6b306f74775fa \ No newline at end of file +e44e26771e5aa48f3bfa14d6db3e669b5b2a1e73 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index bd1e168062..0bdbb0e75b 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -657,6 +657,15 @@ static int multiplexDelete( multiplexFilename(zName, nName, SQLITE_OPEN_MAIN_JOURNAL, --iChunk, z); rc = pOrigVfs->xDelete(pOrigVfs, z, syncDir); } + iChunk = 0; + do{ + multiplexFilename(zName, nName, SQLITE_OPEN_WAL, ++iChunk, z); + rc = pOrigVfs->xAccess(pOrigVfs, z, SQLITE_ACCESS_EXISTS, &bExists); + }while( rc==SQLITE_OK && bExists ); + while( rc==SQLITE_OK && iChunk>1 ){ + multiplexFilename(zName, nName, SQLITE_OPEN_WAL, --iChunk, z); + rc = pOrigVfs->xDelete(pOrigVfs, z, syncDir); + } } sqlite3_free(z); } From 4aced72a59a8331d5b5e8cfcd1310bead5c40fcd Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Feb 2012 11:52:12 +0000 Subject: [PATCH 39/51] Fix a case where an error code was being overwritten in multiplexDelete(). FossilOrigin-Name: c267893a0813beb1764071409025e178318e1ca3 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/test_multiplex.c | 18 ++++++++++-------- test/multiplex3.test | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 0b41ca5008..108aca806a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sxDelete\simplementation\sof\sthe\smultiplexor\sVFS\sso\sthat\sit\scorrectly\ndeletes\soverflow\sWAL\sfiles. -D 2012-02-27T14:28:50.338 +C Fix\sa\scase\swhere\san\serror\scode\swas\sbeing\soverwritten\sin\smultiplexDelete(). +D 2012-02-28T11:52:12.825 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c a6a6baf343f79b942331f13378d045e7e270ae64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c cfe25d74333892ababde61196821a889b4756dee -F src/test_multiplex.c 0b3111a8c1b11963b2ca17a4b47219a5942bc9c0 +F src/test_multiplex.c 0404a61d7795438be5ee5fd3711eed80724df34d F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -613,7 +613,7 @@ F test/misc7.test 6743b810884ef64ae14c07ad1f9f858c40c06100 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a -F test/multiplex3.test 15903c343f1eaa4b00998b7ceacfc4987e4ccfe9 +F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a @@ -990,7 +990,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P d6850667ea61172ea0ebea4a22b6b306f74775fa -R ea71d798aaf411fbd421b77e33a3053e -U drh -Z 3207ff66d565aff181a5c8c10cee009f +P e44e26771e5aa48f3bfa14d6db3e669b5b2a1e73 +R a83bd20b956aa8e93cec00d5fc1e4306 +U dan +Z a4ff6a0dae74370332c0513312e14191 diff --git a/manifest.uuid b/manifest.uuid index e813cbd424..8a835053fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e44e26771e5aa48f3bfa14d6db3e669b5b2a1e73 \ No newline at end of file +c267893a0813beb1764071409025e178318e1ca3 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 0bdbb0e75b..e2b6720652 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -657,14 +657,16 @@ static int multiplexDelete( multiplexFilename(zName, nName, SQLITE_OPEN_MAIN_JOURNAL, --iChunk, z); rc = pOrigVfs->xDelete(pOrigVfs, z, syncDir); } - iChunk = 0; - do{ - multiplexFilename(zName, nName, SQLITE_OPEN_WAL, ++iChunk, z); - rc = pOrigVfs->xAccess(pOrigVfs, z, SQLITE_ACCESS_EXISTS, &bExists); - }while( rc==SQLITE_OK && bExists ); - while( rc==SQLITE_OK && iChunk>1 ){ - multiplexFilename(zName, nName, SQLITE_OPEN_WAL, --iChunk, z); - rc = pOrigVfs->xDelete(pOrigVfs, z, syncDir); + if( rc==SQLITE_OK ){ + iChunk = 0; + do{ + multiplexFilename(zName, nName, SQLITE_OPEN_WAL, ++iChunk, z); + rc = pOrigVfs->xAccess(pOrigVfs, z, SQLITE_ACCESS_EXISTS, &bExists); + }while( rc==SQLITE_OK && bExists ); + while( rc==SQLITE_OK && iChunk>1 ){ + multiplexFilename(zName, nName, SQLITE_OPEN_WAL, --iChunk, z); + rc = pOrigVfs->xDelete(pOrigVfs, z, syncDir); + } } } sqlite3_free(z); diff --git a/test/multiplex3.test b/test/multiplex3.test index 0f52d763be..c1e741acdb 100644 --- a/test/multiplex3.test +++ b/test/multiplex3.test @@ -127,6 +127,39 @@ for {set iTest 1} {$iTest<=100} {incr iTest} { db2 close } +catch { db close } + + +do_test 3.0 { setup_and_save_db } {} +do_faultsim_test 3 -faults ioerr-trans* -prep { + + forcedelete test2.db + set fd [open test2.wal w] + seek $fd 4095 + puts -nonewline $fd x + close $fd + + multiplex_restore_db + sqlite3 db file:test.db?8_3_names=1 + sqlite3 db2 file:test2.db?8_3_names=1 + sqlite3_multiplex_control db main chunk_size [expr 256*1024] + sqlite3_multiplex_control db2 main chunk_size [expr 256*1024] +} -body { + sqlite3_backup B db2 main db main + B step 100000 + set rc [B finish] + if { [string match SQLITE_IOERR_* $rc] } {error "disk I/O error"} + set rc +} -test { + faultsim_test_result {0 SQLITE_OK} + if {$testrc==0} { + set cksum2 [execsql {SELECT md5sum(a, b) FROM t1 ORDER BY a} db2] + if {$cksum2 != $::cksum1} { error "data mismatch" } + } + catch { B finish } + catch { db close } + catch { db2 close } +} catch { db close } sqlite3_multiplex_shutdown From f602963dbc1c8eed4c9fe15b094d97ec74f3825e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Feb 2012 17:57:34 +0000 Subject: [PATCH 40/51] Fix spurious errors that may occur if an empty database is opened and then initialized as a WAL database by a second connection. FossilOrigin-Name: 16330a2f7262173a32ae48a72c0ee2522b6dc554 --- manifest | 15 +++++---- manifest.uuid | 2 +- src/vacuum.c | 20 ++++++----- src/wal.c | 2 +- test/wal8.test | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 test/wal8.test diff --git a/manifest b/manifest index 108aca806a..ffe0e4775e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\swhere\san\serror\scode\swas\sbeing\soverwritten\sin\smultiplexDelete(). -D 2012-02-28T11:52:12.825 +C Fix\sspurious\serrors\sthat\smay\soccur\sif\san\sempty\sdatabase\sis\sopened\sand\sthen\sinitialized\sas\sa\sWAL\sdatabase\sby\sa\ssecond\sconnection. +D 2012-02-28T17:57:34.628 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -239,7 +239,7 @@ F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 906731099c4397bf8adf3fa90a833355e7472af0 -F src/vacuum.c 0c0ba2242355c6048d65e2b333abe0f7c06348fa +F src/vacuum.c bfd53f9bd20a8fdb70b0fa8e77182b866875c0d8 F src/vdbe.c 32720e873ed0a23e6ee928b676cd995864b984d6 F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 @@ -250,7 +250,7 @@ F src/vdbemem.c fb0ac964ccbcd94f595eb993c05bfd9c52468a4a F src/vdbesort.c b25814d385895544ebc8118245c8311ded7f81c9 F src/vdbetrace.c d6e50e04e1ec498150e519058f617d91b8f5c843 F src/vtab.c ab90fb600a3f5e4b7c48d22a4cdb2d6b23239847 -F src/wal.c c164c39e8625b2b6e7d7b2aab5e7296ebd7e8190 +F src/wal.c 7bb3ad807afc7973406c805d5157ec7a2f65e146 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c af623942514571895818b9b7ae11db95ae3b3d88 @@ -913,6 +913,7 @@ F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test f58ed4b8b542f71c7441da12fbd769d99b362437 F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd +F test/wal8.test 5ab217d21f7e5e86af2933a4ffd0d8357cc2c0bd F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test b9f68e39646375c2b877be906babcc15d38b4877 F test/walbig.test 0ab8a430ef420a3114f7092e0f30fc9585ffa155 @@ -990,7 +991,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P e44e26771e5aa48f3bfa14d6db3e669b5b2a1e73 -R a83bd20b956aa8e93cec00d5fc1e4306 +P c267893a0813beb1764071409025e178318e1ca3 +R 751c4f9505bcae82ab2d498f42151225 U dan -Z a4ff6a0dae74370332c0513312e14191 +Z aacac9f6818b59f3fbe792ef77401913 diff --git a/manifest.uuid b/manifest.uuid index 8a835053fa..df6fdce3ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c267893a0813beb1764071409025e178318e1ca3 \ No newline at end of file +16330a2f7262173a32ae48a72c0ee2522b6dc554 \ No newline at end of file diff --git a/src/vacuum.c b/src/vacuum.c index 58a3c9d683..c03b4500b3 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -176,6 +176,18 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ } #endif + rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Begin a transaction and take an exclusive lock on the main database + ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, + ** to ensure that we do not try to change the page-size on a WAL database. + */ + rc = execSql(db, pzErrMsg, "BEGIN;"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = sqlite3BtreeBeginTrans(pMain, 2); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + /* Do not attempt to change the page size for a WAL database */ if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain)) ==PAGER_JOURNALMODE_WAL ){ @@ -189,20 +201,12 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ rc = SQLITE_NOMEM; goto end_of_vacuum; } - rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); - if( rc!=SQLITE_OK ){ - goto end_of_vacuum; - } #ifndef SQLITE_OMIT_AUTOVACUUM sqlite3BtreeSetAutoVacuum(pTemp, db->nextAutovac>=0 ? db->nextAutovac : sqlite3BtreeGetAutoVacuum(pMain)); #endif - /* Begin a transaction */ - rc = execSql(db, pzErrMsg, "BEGIN EXCLUSIVE;"); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - /* Query the schema of the main database. Create a mirror schema ** in the temporary database. */ diff --git a/src/wal.c b/src/wal.c index 97d2185ea2..b077d27d9e 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2397,7 +2397,7 @@ int sqlite3WalRead( iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; *pInWal = 1; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, nOut, iOffset); + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); } *pInWal = 0; diff --git a/test/wal8.test b/test/wal8.test new file mode 100644 index 0000000000..4b97de7ae7 --- /dev/null +++ b/test/wal8.test @@ -0,0 +1,90 @@ +# 2012 February 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. +# +# Specifically, it tests the case where a connection opens an empty +# file. Then, another connection opens the same file and initializes +# the connection as a WAL database. Following this, the first connection +# executes a "PRAGMA page_size = XXX" command to set its expected page +# size, and then queries the database. +# +# This is an unusual case, as normally SQLite is able to glean the page +# size from the database file as soon as it is opened (even before the +# first read transaction is executed), and the "PRAGMA page_size = XXX" +# is a no-op. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix wal8 + +db close +forcedelete test.db test.db-wal + +sqlite3 db test.db +sqlite3 db2 test.db + +do_test 1.0 { + execsql { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } db2 +} {wal} + +do_catchsql_test 1.1 { + PRAGMA page_size = 4096; + VACUUM; +} {0 {}} + +db close +db2 close +forcedelete test.db test.db-wal + +sqlite3 db test.db +sqlite3 db2 test.db + +do_test 2.0 { + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + PRAGMA journal_mode = wal; + } db2 +} {wal} + +do_catchsql_test 2.1 { + PRAGMA page_size = 4096; + VACUUM; +} {0 {}} + +db close +db2 close +forcedelete test.db test.db-wal + +sqlite3 db test.db +sqlite3 db2 test.db + +do_test 3.0 { + execsql { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } db2 +} {wal} + +do_execsql_test 3.1 { + PRAGMA page_size = 4096; + SELECT name FROM sqlite_master; +} {t1} + +finish_test + From 7fcb214b9312710de1fb5050e7fdd632fff387f0 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 Mar 2012 19:44:20 +0000 Subject: [PATCH 41/51] Add the "languageid=" option to fts4. This code is still largely untested and alsmost certainly buggy. FossilOrigin-Name: bea257f70f10dd1111d79cabd1e1462dc651704d --- ext/fts3/fts3.c | 116 +++++++++++++++++++++++-------- ext/fts3/fts3Int.h | 25 ++++--- ext/fts3/fts3_aux.c | 2 +- ext/fts3/fts3_term.c | 2 +- ext/fts3/fts3_write.c | 155 ++++++++++++++++++++++++++++++++---------- manifest | 24 ++++--- manifest.uuid | 2 +- test/fts4langid.test | 78 +++++++++++++++++++++ 8 files changed, 318 insertions(+), 86 deletions(-) create mode 100644 test/fts4langid.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index d17809fe54..6732edaece 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -469,6 +469,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ sqlite3_free(p->zReadExprlist); sqlite3_free(p->zWriteExprlist); sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); @@ -545,7 +546,9 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ int rc; /* Return code */ char *zSql; /* SQL statement passed to declare_vtab() */ char *zCols; /* List of user defined columns */ + const char *zLanguageid; + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); /* Create a list of user columns for the virtual table */ @@ -556,7 +559,8 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ /* Create the whole "CREATE TABLE" statement to pass to SQLite */ zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + zCols, p->zName, zLanguageid ); if( !zCols || !zSql ){ rc = SQLITE_NOMEM; @@ -585,6 +589,7 @@ static int fts3CreateTables(Fts3Table *p){ sqlite3 *db = p->db; /* The database connection */ if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; char *zContentCols; /* Columns of %_content table */ /* Create a list of user columns for the content table */ @@ -593,6 +598,9 @@ static int fts3CreateTables(Fts3Table *p){ char *z = p->azColumn[i]; zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + } if( zContentCols==0 ) rc = SQLITE_NOMEM; /* Create the content table */ @@ -792,6 +800,7 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ for(i=0; inColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); } + if( p->zLanguageid ) fts3Appendf(pRc, &zRet, ",langid"); sqlite3_free(zFree); }else{ fts3Appendf(pRc, &zRet, "rowid"); @@ -799,7 +808,7 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); } } - fts3Appendf(pRc, &zRet, "FROM '%q'.'%q%s' AS x", + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", p->zDb, (p->zContentTbl ? p->zContentTbl : p->zName), (p->zContentTbl ? "" : "_content") @@ -842,6 +851,9 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ for(i=0; inColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } sqlite3_free(zFree); return zRet; } @@ -1057,6 +1069,7 @@ static int fts3InitVtab( char *zCompress = 0; /* compress=? parameter (or NULL) */ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ char *zContent = 0; /* content=? parameter (or NULL) */ + char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) @@ -1106,7 +1119,8 @@ static int fts3InitVtab( { "compress", 8 }, /* 2 -> COMPRESS */ { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ { "order", 5 }, /* 4 -> ORDER */ - { "content", 7 } /* 5 -> CONTENT */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 } /* 6 -> LANGUAGEID */ }; int iOpt; @@ -1160,12 +1174,18 @@ static int fts3InitVtab( bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); break; - default: /* CONTENT */ - assert( iOpt==5 ); - sqlite3_free(zUncompress); + case 5: /* CONTENT */ + sqlite3_free(zContent); zContent = zVal; zVal = 0; break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; } } sqlite3_free(zVal); @@ -1243,7 +1263,9 @@ static int fts3InitVtab( p->bHasStat = isFts4; p->bDescIdx = bDescIdx; p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; zContent = 0; + zLanguageid = 0; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); @@ -1306,6 +1328,7 @@ fts3_init_out: sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free(zContent); + sqlite3_free(zLanguageid); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ @@ -1357,6 +1380,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts3Table *p = (Fts3Table *)pVTab; int i; /* Iterator variable */ int iCons = -1; /* Index of constraint to use */ + int iLangidCons = -1; /* Index of langid=x constraint, if present */ /* By default use a full table scan. This is an expensive option, ** so search through the constraints to see if a more efficient @@ -1369,7 +1393,8 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ if( pCons->usable==0 ) continue; /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + if( iCons<0 + && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 ) ){ pInfo->idxNum = FTS3_DOCID_SEARCH; @@ -1392,7 +1417,13 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; pInfo->estimatedCost = 2.0; iCons = i; - break; + } + + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; } } @@ -1400,6 +1431,9 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->aConstraintUsage[iCons].argvIndex = 1; pInfo->aConstraintUsage[iCons].omit = 1; } + if( iLangidCons>=0 ){ + pInfo->aConstraintUsage[iLangidCons].argvIndex = 2; + } /* Regardless of the strategy selected, FTS can deliver rows in rowid (or ** docid) order. Both ascending and descending are possible. @@ -2549,6 +2583,7 @@ static int fts3SegReaderCursorAppend( */ static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ @@ -2577,7 +2612,7 @@ static int fts3SegReaderCursor( if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); + rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); } while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ @@ -2622,6 +2657,7 @@ static int fts3SegReaderCursor( */ int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iLangid, int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ @@ -2646,7 +2682,7 @@ int sqlite3Fts3SegReaderCursor( memset(pCsr, 0, sizeof(Fts3MultiSegReader)); return fts3SegReaderCursor( - p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr + p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr ); } @@ -2658,11 +2694,14 @@ int sqlite3Fts3SegReaderCursor( */ static int fts3SegReaderCursorAddZero( Fts3Table *p, /* FTS virtual table handle */ + int iLangid, const char *zTerm, /* Term to scan doclist of */ int nTerm, /* Number of bytes in zTerm */ Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */ ){ - return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); + return fts3SegReaderCursor(p, + iLangid, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr + ); } /* @@ -2698,8 +2737,9 @@ static int fts3TermSegReaderCursor( for(i=1; bFound==0 && inIndex; i++){ if( p->aIndex[i].nPrefix==nTerm ){ bFound = 1; - rc = sqlite3Fts3SegReaderCursor( - p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr + ); pSegcsr->bLookup = 1; } } @@ -2707,19 +2747,21 @@ static int fts3TermSegReaderCursor( for(i=1; bFound==0 && inIndex; i++){ if( p->aIndex[i].nPrefix==nTerm+1 ){ bFound = 1; - rc = sqlite3Fts3SegReaderCursor( - p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr ); if( rc==SQLITE_OK ){ - rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr); + rc = fts3SegReaderCursorAddZero( + p, pCsr->iLangid, zTerm, nTerm, pSegcsr + ); } } } } if( bFound==0 ){ - rc = sqlite3Fts3SegReaderCursor( - p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr ); pSegcsr->bLookup = !isPrefix; } @@ -2874,7 +2916,7 @@ static int fts3FilterMethod( UNUSED_PARAMETER(nVal); assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); - assert( nVal==0 || nVal==1 ); + assert( nVal==0 || nVal==1 || nVal==2 ); assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); assert( p->pSegments==0 ); @@ -2910,6 +2952,9 @@ static int fts3FilterMethod( return rc; } + pCsr->iLangid = 0; + if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); + rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; @@ -2971,10 +3016,17 @@ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ /* ** This is the xColumn method, called by SQLite to request a value from ** the row that the supplied cursor currently points to. +** +** If: +** +** (iCol < p->nColumn) -> The value of the iCol'th user column. +** (iCol == p->nColumn) -> Magic column with the same name as the table. +** (iCol == p->nColumn+1) -> Docid column +** (iCol == p->nColumn+2) -> Langid column */ static int fts3ColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ int rc = SQLITE_OK; /* Return Code */ @@ -2982,22 +3034,32 @@ static int fts3ColumnMethod( Fts3Table *p = (Fts3Table *)pCursor->pVtab; /* The column value supplied by SQLite must be in range. */ - assert( iCol>=0 && iCol<=p->nColumn+1 ); + assert( iCol>=0 && iCol<=p->nColumn+2 ); if( iCol==p->nColumn+1 ){ /* This call is a request for the "docid" column. Since "docid" is an ** alias for "rowid", use the xRowid() method to obtain the value. */ - sqlite3_result_int64(pContext, pCsr->iPrevId); + sqlite3_result_int64(pCtx, pCsr->iPrevId); }else if( iCol==p->nColumn ){ /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor. - */ - sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + ** Return a blob which is a pointer to the cursor. */ + sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); }else{ + /* The requested column is either a user column (one that contains + ** indexed data), or the language-id column. */ rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ - sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); + + if( rc==SQLITE_OK ){ + if( iCol==p->nColumn+2 ){ + int iLangid = 0; + if( p->zLanguageid ){ + iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); + } + sqlite3_result_int(pCtx, iLangid); + }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } } } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 16c3de0578..8e889181ca 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -192,6 +192,7 @@ struct Fts3Table { char **azColumn; /* column names. malloced */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. @@ -211,12 +212,12 @@ struct Fts3Table { /* TODO: Fix the first paragraph of this comment. ** - ** The following hash table is used to buffer pending index updates during - ** transactions. Variable nPendingData estimates the memory size of the - ** pending data, including hash table overhead, but not malloc overhead. - ** When nPendingData exceeds nMaxPendingData, the buffer is flushed - ** automatically. Variable iPrevDocid is the docid of the most recently - ** inserted record. + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. Variable nPendingData estimates the memory + ** size of the pending data, including hash table overhead, not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, the buffer + ** is flushed automatically. Variable iPrevDocid is the docid of the most + ** recently inserted record. ** ** A single FTS4 table may have multiple full-text indexes. For each index ** there is an entry in the aIndex[] array. Index 0 is an index of all the @@ -231,12 +232,13 @@ struct Fts3Table { int nMaxPendingData; /* Max pending data before flush to disk */ int nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* State variables used for validating that the transaction control ** methods of the virtual table are called at appropriate times. These - ** values do not contribution to the FTS computation; they are used for - ** verifying the SQLite core. + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. */ int inTransaction; /* True after xBegin but before xCommit/xRollback */ int mxSavepoint; /* Largest valid xSavepoint integer */ @@ -255,6 +257,7 @@ struct Fts3Cursor { u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ int nPhrase; /* Number of matchable phrases in query */ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ @@ -406,7 +409,7 @@ int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, int sqlite3Fts3SegReaderPending( Fts3Table*,int,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); -int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); @@ -427,8 +430,8 @@ int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); -int sqlite3Fts3SegReaderCursor( - Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *); +int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index ada85d796b..89bf3ebf6e 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -376,7 +376,7 @@ static int fts3auxFilterMethod( if( pCsr->zStop==0 ) return SQLITE_NOMEM; } - rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, 0, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index d3eb690bdd..2108fc1251 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -271,7 +271,7 @@ static int fts3termFilterMethod( pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, p->iIndex, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr ); if( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 6d7c3e8d0d..ef2436531c 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -431,6 +431,19 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ return rc; } +static sqlite3_int64 getAbsoluteLevel( + Fts3Table *p, + int iLangid, + int iIndex, + int iLevel +){ + assert( iLangid>=0 ); + assert( p->nIndex>0 ); + assert( iIndex>=0 && iIndexnIndex ); + return (iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL + iLevel; +} + + /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, @@ -450,6 +463,7 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ */ int sqlite3Fts3AllSegdirs( Fts3Table *p, /* FTS3 table */ + int iLangid, /* Language being queried */ int iIndex, /* Index for p->aIndex[] */ int iLevel, /* Level to select */ sqlite3_stmt **ppStmt /* OUT: Compiled statement */ @@ -465,8 +479,10 @@ int sqlite3Fts3AllSegdirs( /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); - sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); } }else{ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ @@ -714,18 +730,28 @@ static int fts3PendingTermsAdd( ** fts3PendingTermsAdd() are to add term/position-list pairs for the ** contents of the document with docid iDocid. */ -static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ +static int fts3PendingTermsDocid( + Fts3Table *p, /* Full-text table handle */ + int iLangid, /* Language id of row being written */ + sqlite_int64 iDocid /* Docid of row being written */ +){ + assert( iLangid>=0 ); + /* TODO(shess) Explore whether partially flushing the buffer on ** forced-flush would provide better performance. I suspect that if ** we ordered the doclists by size and flushed the largest until the ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ - if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){ + if( iDocid<=p->iPrevDocid + || p->iPrevLangid!=iLangid + || p->nPendingData>p->nMaxPendingData + ){ int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; } p->iPrevDocid = iDocid; + p->iPrevLangid = iLangid; return SQLITE_OK; } @@ -779,6 +805,7 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ ** apVal[p->nColumn+1] Right-most user-defined column ** apVal[p->nColumn+2] Hidden column with same name as table ** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +** apVal[p->nColumn+4] Hidden languageid column */ static int fts3InsertData( Fts3Table *p, /* Full-text table */ @@ -809,9 +836,13 @@ static int fts3InsertData( ** defined columns in the FTS3 table, plus one for the docid field. */ rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); - if( rc!=SQLITE_OK ){ - return rc; + if( rc==SQLITE_OK && p->zLanguageid ){ + rc = sqlite3_bind_int( + pContentInsert, p->nColumn+2, + sqlite3_value_int(apVal[p->nColumn+4]) + ); } + if( rc!=SQLITE_OK ) return rc; /* There is a quirk here. The users INSERT statement may have specified ** a value for the "rowid" field, for the "docid" field, or for both. @@ -871,6 +902,15 @@ static int fts3DeleteAll(Fts3Table *p, int bContent){ return rc; } +/* +** +*/ +static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ + int iLangid = 0; + if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); + return iLangid; +} + /* ** The first element in the apVal[] array is assumed to contain the docid ** (an integer) of a row about to be deleted. Remove all terms from the @@ -890,16 +930,20 @@ static void fts3DeleteTerms( if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; - for(i=1; i<=p->nColumn; i++){ + rc = fts3PendingTermsDocid(p, + langidFromSelect(p, pSelect), + sqlite3_column_int64(pSelect, 0) + ); + for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ const char *zText = (const char *)sqlite3_column_text(pSelect, i); rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]); - if( rc!=SQLITE_OK ){ - sqlite3_reset(pSelect); - *pRC = rc; - return; - } aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); } + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + *pRC = rc; + return; + } } rc = sqlite3_reset(pSelect); }else{ @@ -912,7 +956,7 @@ static void fts3DeleteTerms( ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ -static int fts3SegmentMerge(Fts3Table *, int, int); +static int fts3SegmentMerge(Fts3Table *, int, int, int); /* ** This function allocates a new level iLevel index in the segdir table. @@ -931,6 +975,7 @@ static int fts3SegmentMerge(Fts3Table *, int, int); */ static int fts3AllocateSegdirIdx( Fts3Table *p, + int iLangid, /* Language id */ int iIndex, /* Index for p->aIndex */ int iLevel, int *piIdx @@ -939,10 +984,15 @@ static int fts3AllocateSegdirIdx( sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext = 0; /* Result of query pNextIdx */ + assert( iLangid>=0 ); + assert( p->nIndex>=1 ); + /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + sqlite3_bind_int64( + pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int(pNextIdx, 0); } @@ -956,7 +1006,7 @@ static int fts3AllocateSegdirIdx( ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - rc = fts3SegmentMerge(p, iIndex, iLevel); + rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); *piIdx = 0; }else{ *piIdx = iNext; @@ -2216,6 +2266,7 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ + int iLangid, /* Language id */ int iIndex, /* Index for p->aIndex */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ @@ -2243,13 +2294,15 @@ static int fts3DeleteSegdir( if( iLevel==FTS3_SEGCURSOR_ALL ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); - sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); + sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int(pDelete, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); } }else{ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); } } @@ -2718,13 +2771,18 @@ void sqlite3Fts3SegReaderFinish( ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ -static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ +static int fts3SegmentMerge( + Fts3Table *p, + int iLangid, /* Language id to merge */ + int iIndex, /* Index in p->aIndex[] to merge */ + int iLevel /* Level to merge */ +){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ int iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ - Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ assert( iLevel==FTS3_SEGCURSOR_ALL @@ -2734,7 +2792,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ assert( iLevel=0 && iIndexnIndex ); - rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); + rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; if( iLevel==FTS3_SEGCURSOR_ALL ){ @@ -2750,20 +2808,20 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ bIgnoreEmpty = 1; }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; - rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); - iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); - assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); - assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); + assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert( iNewLevelnIndex; i++){ - rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); + rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); @@ -2954,12 +3014,16 @@ static void fts3UpdateDocTotals( sqlite3_free(a); } +/* +** Merge the entire database so that there is one segment for each +** iIndex/iLangid combination. +*/ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ int i; int bSeenDone = 0; int rc = SQLITE_OK; for(i=0; rc==SQLITE_OK && inIndex; i++){ - rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); + rc = fts3SegmentMerge(p, 0, i, FTS3_SEGCURSOR_ALL); if( rc==SQLITE_DONE ){ bSeenDone = 1; rc = SQLITE_OK; @@ -3015,7 +3079,9 @@ static int fts3DoRebuild(Fts3Table *p){ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ int iCol; - rc = fts3PendingTermsDocid(p, sqlite3_column_int64(pStmt, 0)); + rc = fts3PendingTermsDocid(p, + langidFromSelect(p, pStmt), sqlite3_column_int64(pStmt, 0) + ); aSz[p->nColumn] = 0; for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); @@ -3245,8 +3311,6 @@ static int fts3DeleteByRowid( rc = fts3DeleteAll(p, 1); *pnDoc = *pnDoc - 1; }else{ - sqlite3_int64 iRemove = sqlite3_value_int64(pRowid); - rc = fts3PendingTermsDocid(p, iRemove); fts3DeleteTerms(&rc, p, pRowid, aSzDel); if( p->zContentTbl==0 ){ fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); @@ -3265,7 +3329,16 @@ static int fts3DeleteByRowid( /* ** This function does the work for the xUpdate method of FTS3 virtual -** tables. +** tables. The schema of the virtual table being: +** +** CREATE TABLE ( +** , +**
HIDDEN, +** docid HIDDEN, +** HIDDEN +** ); +** +** */ int sqlite3Fts3UpdateMethod( sqlite3_vtab *pVtab, /* FTS3 vtab object */ @@ -3282,6 +3355,10 @@ int sqlite3Fts3UpdateMethod( int bInsertDone = 0; assert( p->pSegments==0 ); + assert( + nArg==1 /* DELETE operations */ + || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ + ); /* Check for a "special" INSERT operation. One of the form: ** @@ -3295,6 +3372,11 @@ int sqlite3Fts3UpdateMethod( goto update_out; } + if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ + rc = SQLITE_CONSTRAINT; + goto update_out; + } + /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); if( aSzIns==0 ){ @@ -3369,7 +3451,10 @@ int sqlite3Fts3UpdateMethod( } } if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ - rc = fts3PendingTermsDocid(p, *pRowid); + rc = fts3PendingTermsDocid(p, + sqlite3_value_int(apVal[2 + p->nColumn + 2]), + *pRowid + ); } if( rc==SQLITE_OK ){ assert( p->iPrevDocid==*pRowid ); diff --git a/manifest b/manifest index ffe0e4775e..c4b93f56a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sspurious\serrors\sthat\smay\soccur\sif\san\sempty\sdatabase\sis\sopened\sand\sthen\sinitialized\sas\sa\sWAL\sdatabase\sby\sa\ssecond\sconnection. -D 2012-02-28T17:57:34.628 +C Add\sthe\s"languageid="\soption\sto\sfts4.\sThis\scode\sis\sstill\slargely\suntested\sand\salsmost\scertainly\sbuggy. +D 2012-03-01T19:44:20.362 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,22 +63,22 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 4cf7b8e5bbb6667f5d7818fa0bf064fbbb72b086 +F ext/fts3/fts3.c 93a8eb6e6eb4cd0aa4856d841a9d8d0025a2784a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h ce958a6fa92a95462853aa3acc0b69bcda39102f -F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 +F ext/fts3/fts3Int.h 8ba2d8ce5db6da67c5e5e7b8a0b90e6d80826546 +F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239 F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa F ext/fts3/fts3_porter.c b7e5276f9f0a5fc7018b6fa55ce0f31f269ef881 F ext/fts3/fts3_snippet.c 1f9ee6a8e0e242649645968dcec4deb253d86c2a -F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef +F ext/fts3/fts3_term.c d3466cf99432291be08e379d89645462431809d6 F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 1721187a4dec29ef9ae648ad8478da741085af18 +F ext/fts3/fts3_write.c 489d262b1ee9ab1dbb4da48bd8737fac15d0f58f F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -496,6 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f +F test/fts4langid.test 3d968b7c0afb8be1609794267f34b89d378a81ea F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -991,7 +992,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P c267893a0813beb1764071409025e178318e1ca3 -R 751c4f9505bcae82ab2d498f42151225 +P 16330a2f7262173a32ae48a72c0ee2522b6dc554 +R 86036df8ba11902f17395620671e5794 +T *branch * fts4-languageid +T *sym-fts4-languageid * +T -sym-trunk * U dan -Z aacac9f6818b59f3fbe792ef77401913 +Z 6902c01b6e8a000d5e06f8fe8778490f diff --git a/manifest.uuid b/manifest.uuid index df6fdce3ee..d2201717b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16330a2f7262173a32ae48a72c0ee2522b6dc554 \ No newline at end of file +bea257f70f10dd1111d79cabd1e1462dc651704d \ No newline at end of file diff --git a/test/fts4langid.test b/test/fts4langid.test new file mode 100644 index 0000000000..a65ccedac3 --- /dev/null +++ b/test/fts4langid.test @@ -0,0 +1,78 @@ +# 2012 March 01 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the languageid=xxx FTS4 option. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix fts4content + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +set ::testprefix fts4langid + + + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=lang_id); +} + +do_execsql_test 1.2 { + SELECT sql FROM sqlite_master WHERE name = 't1_content'; +} {{CREATE TABLE 't1_content'(docid INTEGER PRIMARY KEY, 'c0a', 'c1b', langid)}} + +do_execsql_test 1.3 {SELECT docid FROM t1} {} +do_execsql_test 1.4 {SELECT lang_id FROM t1} {} + +do_execsql_test 1.5 {INSERT INTO t1(a, b) VALUES('aaa', 'bbb')} +do_execsql_test 1.6 {SELECT lang_id FROM t1 } {0} + +do_execsql_test 1.7 {INSERT INTO t1(a, b, lang_id) VALUES('aaa', 'bbb', 4)} +do_execsql_test 1.8 {SELECT lang_id FROM t1 } {0 4} + +do_execsql_test 1.9 {INSERT INTO t1(a, b, lang_id) VALUES('aaa', 'bbb', 'xyz')} +do_execsql_test 1.10 {SELECT lang_id FROM t1} {0 4 0} + +do_execsql_test 1.11 { + CREATE VIRTUAL TABLE t2 USING fts4; + INSERT INTO t2 VALUES('abc'); +} +do_execsql_test 1.12 { SELECT rowid FROM t2 WHERE content MATCH 'abc' } 1 + +do_execsql_test 1.13 { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts4(languageid=lang_id); + INSERT INTO t1(content) VALUES('a b c'); + INSERT INTO t1(content, lang_id) VALUES('a b c', 1); +} + +do_execsql_test 1.14 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b'; +} {1} +do_execsql_test 1.15 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b' AND lang_id = 0; +} {1} + +do_execsql_test 1.16 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b' AND lang_id = 1; +} {2} + +do_catchsql_test 1.17 { + INSERT INTO t1(content, lang_id) VALUES('123', -1); +} {1 {constraint failed}} + +finish_test + From cd565fd118219d1b979672e1710275ea87c85c07 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 Mar 2012 21:30:44 +0000 Subject: [PATCH 42/51] Fix a typo in the header comment of sqlite3_commit_hook(). FossilOrigin-Name: 3784d1475b49f24e4a51450edc94aee66fd48fe4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 5 ++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index ffe0e4775e..1bfda8c87a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sspurious\serrors\sthat\smay\soccur\sif\san\sempty\sdatabase\sis\sopened\sand\sthen\sinitialized\sas\sa\sWAL\sdatabase\sby\sa\ssecond\sconnection. -D 2012-02-28T17:57:34.628 +C Fix\sa\stypo\sin\sthe\sheader\scomment\sof\ssqlite3_commit_hook(). +D 2012-03-01T21:30:44.361 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -147,7 +147,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d -F src/main.c e936269fbc1af78cef79c27980e89209aeb20a07 +F src/main.c dcb4a15254dca9cc59dba63e813a8c140c021832 F src/malloc.c 15afac5e59b6584efe072e9933aefb4230e74f97 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c b3677415e69603d6a0e7c5410a1b3731d55beda1 @@ -991,7 +991,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P c267893a0813beb1764071409025e178318e1ca3 -R 751c4f9505bcae82ab2d498f42151225 -U dan -Z aacac9f6818b59f3fbe792ef77401913 +P 16330a2f7262173a32ae48a72c0ee2522b6dc554 +R 4fca0028f2954d371f8ed29b9be73d18 +U drh +Z a58e0a8dbfee9129245f4f54328583b6 diff --git a/manifest.uuid b/manifest.uuid index df6fdce3ee..7506854a4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16330a2f7262173a32ae48a72c0ee2522b6dc554 \ No newline at end of file +3784d1475b49f24e4a51450edc94aee66fd48fe4 \ No newline at end of file diff --git a/src/main.c b/src/main.c index dedfba4a19..3a1dff576a 100644 --- a/src/main.c +++ b/src/main.c @@ -1312,9 +1312,8 @@ void *sqlite3_profile( } #endif /* SQLITE_OMIT_TRACE */ -/*** EXPERIMENTAL *** -** -** Register a function to be invoked when a transaction comments. +/* +** Register a function to be invoked when a transaction commits. ** If the invoked function returns non-zero, then the commit becomes a ** rollback. */ From e3ddd1ac42834a0189fa359a32755c2e9b733224 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Mar 2012 11:48:50 +0000 Subject: [PATCH 43/51] Fix a bug in merging FTS language tables for languages other than language 0. FossilOrigin-Name: d281cb8984c911a4c0cce2ec299e1351d8e580e4 --- ext/fts3/fts3_write.c | 2 +- manifest | 17 ++-- manifest.uuid | 2 +- test/fts4langid.test | 181 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 190 insertions(+), 12 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index ef2436531c..b158cc4510 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -488,7 +488,7 @@ int sqlite3Fts3AllSegdirs( /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); } } *ppStmt = pStmt; diff --git a/manifest b/manifest index c4b93f56a0..e051362a62 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"languageid="\soption\sto\sfts4.\sThis\scode\sis\sstill\slargely\suntested\sand\salsmost\scertainly\sbuggy. -D 2012-03-01T19:44:20.362 +C Fix\sa\sbug\sin\smerging\sFTS\slanguage\stables\sfor\slanguages\sother\sthan\slanguage\s0. +D 2012-03-02T11:48:50.564 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 489d262b1ee9ab1dbb4da48bd8737fac15d0f58f +F ext/fts3/fts3_write.c 36fc2e3a28f51ee135a344877c1e4be0a9f45e6e F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f -F test/fts4langid.test 3d968b7c0afb8be1609794267f34b89d378a81ea +F test/fts4langid.test 7ab7be619d3acb3727e4bef3230ba3dbcf2e0556 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -992,10 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 16330a2f7262173a32ae48a72c0ee2522b6dc554 -R 86036df8ba11902f17395620671e5794 -T *branch * fts4-languageid -T *sym-fts4-languageid * -T -sym-trunk * +P bea257f70f10dd1111d79cabd1e1462dc651704d +R a3a9247d2c76c9d90f9fc486f3311f0d U dan -Z 6902c01b6e8a000d5e06f8fe8778490f +Z f1e998b56e58f712fe6da1411961b8ef diff --git a/manifest.uuid b/manifest.uuid index d2201717b8..ce8a1c3b82 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bea257f70f10dd1111d79cabd1e1462dc651704d \ No newline at end of file +d281cb8984c911a4c0cce2ec299e1351d8e580e4 \ No newline at end of file diff --git a/test/fts4langid.test b/test/fts4langid.test index a65ccedac3..51c42ddb54 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -24,6 +24,27 @@ ifcapable !fts3 { set ::testprefix fts4langid +#--------------------------------------------------------------------------- +# Test plan: +# +# 1.* - Warm-body tests created for specific purposes during development. +# Passing these doesn't really prove much. +# +# 2.* - Test that FTS queries only ever return rows associated with +# the requested language. +# +# 3.* - Test that the 'optimize' and 'rebuild' commands work correctly. +# +# 4.* - Test that if one is provided, the tokenizer xLanguage method +# is called to configure the tokenizer before tokenizing query +# or document text. +# +# 5.* - Test the fts4aux table when the associated FTS4 table contains +# multiple languages. +# +# 6.* - Tests with content= tables. Both where there is a real +# underlying content table and where there is not. +# do_execsql_test 1.1 { @@ -74,5 +95,165 @@ do_catchsql_test 1.17 { INSERT INTO t1(content, lang_id) VALUES('123', -1); } {1 {constraint failed}} +do_execsql_test 1.18 { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts4(languageid=lang_id); + INSERT INTO t1(content, lang_id) VALUES('A', 13); + INSERT INTO t1(content, lang_id) VALUES('B', 13); + INSERT INTO t1(content, lang_id) VALUES('C', 13); + INSERT INTO t1(content, lang_id) VALUES('D', 13); + INSERT INTO t1(content, lang_id) VALUES('E', 13); + INSERT INTO t1(content, lang_id) VALUES('F', 13); + INSERT INTO t1(content, lang_id) VALUES('G', 13); + INSERT INTO t1(content, lang_id) VALUES('H', 13); + INSERT INTO t1(content, lang_id) VALUES('I', 13); + INSERT INTO t1(content, lang_id) VALUES('J', 13); + INSERT INTO t1(content, lang_id) VALUES('K', 13); + INSERT INTO t1(content, lang_id) VALUES('L', 13); + INSERT INTO t1(content, lang_id) VALUES('M', 13); + INSERT INTO t1(content, lang_id) VALUES('N', 13); + INSERT INTO t1(content, lang_id) VALUES('O', 13); + INSERT INTO t1(content, lang_id) VALUES('P', 13); + INSERT INTO t1(content, lang_id) VALUES('Q', 13); + INSERT INTO t1(content, lang_id) VALUES('R', 13); + INSERT INTO t1(content, lang_id) VALUES('S', 13); + SELECT rowid FROM t1 WHERE t1 MATCH 'A'; +} {} + + +#------------------------------------------------------------------------- +# Test cases 2.* +# + +proc build_multilingual_db_1 {db} { + $db eval { CREATE VIRTUAL TABLE t2 USING fts4(x, y, languageid=l) } + + set xwords [list zero one two three four five six seven eight nine ten] + set ywords [list alpha beta gamma delta epsilon zeta eta theta iota kappa] + + for {set i 0} {$i < 1000} {incr i} { + set iLangid [expr $i%9] + set x "" + set y "" + + set x [list] + lappend x [lindex $xwords [expr ($i / 1000) % 10]] + lappend x [lindex $xwords [expr ($i / 100) % 10]] + lappend x [lindex $xwords [expr ($i / 10) % 10]] + lappend x [lindex $xwords [expr ($i / 1) % 10]] + + set y [list] + lappend y [lindex $ywords [expr ($i / 1000) % 10]] + lappend y [lindex $ywords [expr ($i / 100) % 10]] + lappend y [lindex $ywords [expr ($i / 10) % 10]] + lappend y [lindex $ywords [expr ($i / 1) % 10]] + + $db eval { INSERT INTO t2(docid, x, y, l) VALUES($i, $x, $y, $iLangid) } + } +} + +proc rowid_list_set_langid {langid} { + set ::rowid_list_langid $langid +} +proc rowid_list {pattern} { + set langid $::rowid_list_langid + set res [list] + db eval {SELECT docid, x, y FROM t2 WHERE l = $langid ORDER BY docid ASC} { + if {[string match "*$pattern*" $x] || [string match "*$pattern*" $y]} { + lappend res $docid + } + } + return $res +} + +proc or_merge_list {list1 list2} { + set res [list] + + set i1 0 + set i2 0 + + set n1 [llength $list1] + set n2 [llength $list2] + + while {$i1 < $n1 && $i2 < $n2} { + set e1 [lindex $list1 $i1] + set e2 [lindex $list2 $i2] + + if {$e1==$e2} { + lappend res $e1 + incr i1 + incr i2 + } elseif {$e1 < $e2} { + lappend res $e1 + incr i1 + } else { + lappend res $e2 + incr i2 + } + } + + concat $res [lrange $list1 $i1 end] [lrange $list2 $i2 end] +} + +proc or_merge_lists {args} { + set res [lindex $args 0] + for {set i 1} {$i < [llength $args]} {incr i} { + set res [or_merge_list $res [lindex $args $i]] + } + set res +} + +proc and_merge_list {list1 list2} { + foreach i $list2 { set a($i) 1 } + set res [list] + foreach i $list1 { + if {[info exists a($i)]} {lappend res $i} + } + set res +} + + +proc and_merge_lists {args} { + set res [lindex $args 0] + for {set i 1} {$i < [llength $args]} {incr i} { + set res [and_merge_list $res [lindex $args $i]] + } + set res +} + +proc filter_list {list langid} { + set res [list] + foreach i $list { + if {($i % 9) == $langid} {lappend res $i} + } + set res +} + +do_test 2.0 { + reset_db + build_multilingual_db_1 db +} {} + +proc do_test_2.1 {tn query res_script} { + for {set langid 0} {$langid < 10} {incr langid} { + rowid_list_set_langid $langid + set res [eval $res_script] + + set actual [ + execsql {SELECT docid FROM t2 WHERE t2 MATCH $query AND l = $langid} + ] + do_test 2.1.$tn.$langid [list set {} $actual] $res + } +} + +do_test_2.1 1 {delta} { rowid_list delta } +do_test_2.1 2 {"zero one two"} { rowid_list "zero one two" } +do_test_2.1 3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_2.1 4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + finish_test From 8fb662b0beb67f2400f1747283f9bfc7cbc0d0a9 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Mar 2012 12:26:19 +0000 Subject: [PATCH 44/51] Fix the FTS 'optimize' command on multi-lingual databases. FossilOrigin-Name: 65fa693729a336e4d905ce72e6b9ccf4faa772bd --- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_write.c | 45 ++++++++++++++++++++++++++++++++----------- manifest | 16 +++++++-------- manifest.uuid | 2 +- test/fts4langid.test | 28 +++++++++++++++++++++------ 5 files changed, 66 insertions(+), 27 deletions(-) diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 8e889181ca..078b5b987b 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -197,7 +197,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[27]; + sqlite3_stmt *aStmt[28]; char *zReadExprlist; char *zWriteExprlist; diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index b158cc4510..decbe0b73a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -232,6 +232,8 @@ struct SegmentNode { #define SQL_DELETE_SEGDIR_RANGE 26 +#define SQL_SELECT_ALL_LANGID 27 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -285,6 +287,7 @@ static int fts3SqlStmt( /* 25 */ "", /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", +/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'", }; int rc = SQLITE_OK; @@ -2229,7 +2232,12 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ +static int fts3SegmentMaxLevel( + Fts3Table *p, + int iLangid, + int iIndex, + int *pnMax +){ sqlite3_stmt *pStmt; int rc; assert( iIndex>=0 && iIndexnIndex ); @@ -2242,8 +2250,10 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); - sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); + sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int(pStmt, 0); } @@ -2804,7 +2814,7 @@ static int fts3SegmentMerge( rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel); bIgnoreEmpty = 1; }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ @@ -3019,16 +3029,29 @@ static void fts3UpdateDocTotals( ** iIndex/iLangid combination. */ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ - int i; int bSeenDone = 0; - int rc = SQLITE_OK; - for(i=0; rc==SQLITE_OK && inIndex; i++){ - rc = fts3SegmentMerge(p, 0, i, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_DONE ){ - bSeenDone = 1; - rc = SQLITE_OK; + int rc; + sqlite3_stmt *pAllLangid = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->nIndex); + while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int i; + int iLangid = sqlite3_column_int(pAllLangid, 0); + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; } + sqlite3Fts3SegmentsClose(p); sqlite3Fts3PendingTermsClear(p); diff --git a/manifest b/manifest index e051362a62..60424716c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\smerging\sFTS\slanguage\stables\sfor\slanguages\sother\sthan\slanguage\s0. -D 2012-03-02T11:48:50.564 +C Fix\sthe\sFTS\s'optimize'\scommand\son\smulti-lingual\sdatabases. +D 2012-03-02T12:26:19.396 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -65,7 +65,7 @@ F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c 93a8eb6e6eb4cd0aa4856d841a9d8d0025a2784a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 8ba2d8ce5db6da67c5e5e7b8a0b90e6d80826546 +F ext/fts3/fts3Int.h 521d300f2af4e741f53c4e2dd540275fb64533eb F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239 F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 36fc2e3a28f51ee135a344877c1e4be0a9f45e6e +F ext/fts3/fts3_write.c 35b98a42f9bbdd28af1b1f3bb0c09ff07090a764 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f -F test/fts4langid.test 7ab7be619d3acb3727e4bef3230ba3dbcf2e0556 +F test/fts4langid.test a793f2da4cbe9c8ad2f49d2a013c6a0ff61e1783 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P bea257f70f10dd1111d79cabd1e1462dc651704d -R a3a9247d2c76c9d90f9fc486f3311f0d +P d281cb8984c911a4c0cce2ec299e1351d8e580e4 +R 2e9c73a6dc49c268fdce6f613b343e49 U dan -Z f1e998b56e58f712fe6da1411961b8ef +Z bf6a4c44b9753bccc8cfbcc7423214ad diff --git a/manifest.uuid b/manifest.uuid index ce8a1c3b82..9089612df2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d281cb8984c911a4c0cce2ec299e1351d8e580e4 \ No newline at end of file +65fa693729a336e4d905ce72e6b9ccf4faa772bd \ No newline at end of file diff --git a/test/fts4langid.test b/test/fts4langid.test index 51c42ddb54..310332d650 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -234,7 +234,7 @@ do_test 2.0 { build_multilingual_db_1 db } {} -proc do_test_2.1 {tn query res_script} { +proc do_test_2 {tn query res_script} { for {set langid 0} {$langid < 10} {incr langid} { rowid_list_set_langid $langid set res [eval $res_script] @@ -242,16 +242,32 @@ proc do_test_2.1 {tn query res_script} { set actual [ execsql {SELECT docid FROM t2 WHERE t2 MATCH $query AND l = $langid} ] - do_test 2.1.$tn.$langid [list set {} $actual] $res + do_test 2.$tn.$langid [list set {} $actual] $res } } -do_test_2.1 1 {delta} { rowid_list delta } -do_test_2.1 2 {"zero one two"} { rowid_list "zero one two" } -do_test_2.1 3 {zero one two} { +# Run some queries. +do_test_2 1.1 {delta} { rowid_list delta } +do_test_2 1.2 {"zero one two"} { rowid_list "zero one two" } +do_test_2 1.3 {zero one two} { and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] } -do_test_2.1 4 {"zero one" OR "one two"} { +do_test_2 1.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + +# Now try the same tests as above, but after running the 'optimize' +# command on the FTS table. +do_execsql_test 2.2 { + INSERT INTO t2(t2) VALUES('optimize'); + SELECT count(*) FROM t2_segdir; +} {9} +do_test_2 2.1 {delta} { rowid_list delta } +do_test_2 2.2 {"zero one two"} { rowid_list "zero one two" } +do_test_2 2.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_2 2.4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } From c0db214b107baa92e4985551c8431eb15b3d3261 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Mar 2012 16:18:21 +0000 Subject: [PATCH 45/51] Add test for FTS 'rebuild' command. FossilOrigin-Name: 181bc35731f19c3e4497ba3338c209918d34ea69 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fts4langid.test | 40 +++++++++++++++++++++++++++++----------- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 60424716c2..268d6f64f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sFTS\s'optimize'\scommand\son\smulti-lingual\sdatabases. -D 2012-03-02T12:26:19.396 +C Add\stest\sfor\sFTS\s'rebuild'\scommand. +D 2012-03-02T16:18:21.845 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f -F test/fts4langid.test a793f2da4cbe9c8ad2f49d2a013c6a0ff61e1783 +F test/fts4langid.test 343a65d54419b45a318a1acdb1c37749f26fdc79 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P d281cb8984c911a4c0cce2ec299e1351d8e580e4 -R 2e9c73a6dc49c268fdce6f613b343e49 +P 65fa693729a336e4d905ce72e6b9ccf4faa772bd +R 4e2c13b79ff592d73e478288d906b721 U dan -Z bf6a4c44b9753bccc8cfbcc7423214ad +Z bcf6ed8432a11d3fdd53825eba08d4d7 diff --git a/manifest.uuid b/manifest.uuid index 9089612df2..bdb54dd27f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65fa693729a336e4d905ce72e6b9ccf4faa772bd \ No newline at end of file +181bc35731f19c3e4497ba3338c209918d34ea69 \ No newline at end of file diff --git a/test/fts4langid.test b/test/fts4langid.test index 310332d650..059a56d08f 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -30,19 +30,21 @@ set ::testprefix fts4langid # 1.* - Warm-body tests created for specific purposes during development. # Passing these doesn't really prove much. # -# 2.* - Test that FTS queries only ever return rows associated with -# the requested language. +# 2.1.* - Test that FTS queries only ever return rows associated with +# the requested language. # -# 3.* - Test that the 'optimize' and 'rebuild' commands work correctly. +# 2.2.* - Same as 2.1.*, after an 'optimize' command. # -# 4.* - Test that if one is provided, the tokenizer xLanguage method +# 2.3.* - Same as 2.1.*, after a 'rebuild' command. +# +# 3.* - Test that if one is provided, the tokenizer xLanguage method # is called to configure the tokenizer before tokenizing query # or document text. # -# 5.* - Test the fts4aux table when the associated FTS4 table contains +# 4.* - Test the fts4aux table when the associated FTS4 table contains # multiple languages. # -# 6.* - Tests with content= tables. Both where there is a real +# 5.* - Tests with content= tables. Both where there is a real # underlying content table and where there is not. # @@ -124,7 +126,6 @@ do_execsql_test 1.18 { #------------------------------------------------------------------------- # Test cases 2.* # - proc build_multilingual_db_1 {db} { $db eval { CREATE VIRTUAL TABLE t2 USING fts4(x, y, languageid=l) } @@ -258,16 +259,33 @@ do_test_2 1.4 {"zero one" OR "one two"} { # Now try the same tests as above, but after running the 'optimize' # command on the FTS table. +# do_execsql_test 2.2 { INSERT INTO t2(t2) VALUES('optimize'); SELECT count(*) FROM t2_segdir; } {9} -do_test_2 2.1 {delta} { rowid_list delta } -do_test_2 2.2 {"zero one two"} { rowid_list "zero one two" } -do_test_2 2.3 {zero one two} { +do_test_2 2.1 {delta} { rowid_list delta } +do_test_2 2.2 {"zero one two"} { rowid_list "zero one two" } +do_test_2 2.3 {zero one two} { and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] } -do_test_2 2.4 {"zero one" OR "one two"} { +do_test_2 2.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + +# And rebuild. +# +do_test 2.3 { + reset_db + build_multilingual_db_1 db + execsql { INSERT INTO t2(t2) VALUES('rebuild') } +} {} +do_test_2 3.1 {delta} { rowid_list delta } +do_test_2 3.2 {"zero one two"} { rowid_list "zero one two" } +do_test_2 3.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_2 3.4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } From 7395599031f856ac889a575a6192bb0e756af04e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Mar 2012 19:53:02 +0000 Subject: [PATCH 46/51] Fix problems with combining content= and languageid= in a single fts4 table. FossilOrigin-Name: 22491e7bc38aee43819b888e04241cb6a6ef73a3 --- ext/fts3/fts3.c | 23 +++++++++- manifest | 14 +++---- manifest.uuid | 2 +- test/fts4langid.test | 99 ++++++++++++++++++++++++++++++++++---------- 4 files changed, 106 insertions(+), 32 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 6732edaece..dec7f8722c 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -800,13 +800,18 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ for(i=0; inColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); } - if( p->zLanguageid ) fts3Appendf(pRc, &zRet, ",langid"); + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } sqlite3_free(zFree); }else{ fts3Appendf(pRc, &zRet, "rowid"); for(i=0; inColumn; i++){ fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } } fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", p->zDb, @@ -1215,8 +1220,20 @@ static int fts3InitVtab( sqlite3_free((void*)aCol); aCol = 0; rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; j0 ); } if( rc!=SQLITE_OK ) goto fts3_init_out; @@ -3045,6 +3062,8 @@ static int fts3ColumnMethod( /* The extra column whose name is the same as the table. ** Return a blob which is a pointer to the cursor. */ sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); }else{ /* The requested column is either a user column (one that contains ** indexed data), or the language-id column. */ diff --git a/manifest b/manifest index 268d6f64f6..15fcd20c9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sfor\sFTS\s'rebuild'\scommand. -D 2012-03-02T16:18:21.845 +C Fix\sproblems\swith\scombining\scontent=\sand\slanguageid=\sin\sa\ssingle\sfts4\stable. +D 2012-03-02T19:53:02.350 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,7 +63,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 93a8eb6e6eb4cd0aa4856d841a9d8d0025a2784a +F ext/fts3/fts3.c fd89caa4169520c32cf46ca5a62df6dd48201422 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 521d300f2af4e741f53c4e2dd540275fb64533eb F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239 @@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f -F test/fts4langid.test 343a65d54419b45a318a1acdb1c37749f26fdc79 +F test/fts4langid.test be989b5cddcd7596b87232af193f6c4560a34272 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 65fa693729a336e4d905ce72e6b9ccf4faa772bd -R 4e2c13b79ff592d73e478288d906b721 +P 181bc35731f19c3e4497ba3338c209918d34ea69 +R acfb20f690a18ac8d67e116ae8c76f7d U dan -Z bcf6ed8432a11d3fdd53825eba08d4d7 +Z b771fbbc3f5dd49e39970a5f917fc8b7 diff --git a/manifest.uuid b/manifest.uuid index bdb54dd27f..def5beae28 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -181bc35731f19c3e4497ba3338c209918d34ea69 \ No newline at end of file +22491e7bc38aee43819b888e04241cb6a6ef73a3 \ No newline at end of file diff --git a/test/fts4langid.test b/test/fts4langid.test index 059a56d08f..b9110cb768 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -37,17 +37,17 @@ set ::testprefix fts4langid # # 2.3.* - Same as 2.1.*, after a 'rebuild' command. # -# 3.* - Test that if one is provided, the tokenizer xLanguage method +# 3.* - Tests with content= tables. Both where there is a real +# underlying content table and where there is not. +# +# +# 4.* - Test that if one is provided, the tokenizer xLanguage method # is called to configure the tokenizer before tokenizing query # or document text. # -# 4.* - Test the fts4aux table when the associated FTS4 table contains +# 5.* - Test the fts4aux table when the associated FTS4 table contains # multiple languages. # -# 5.* - Tests with content= tables. Both where there is a real -# underlying content table and where there is not. -# - do_execsql_test 1.1 { CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=lang_id); @@ -151,6 +151,11 @@ proc build_multilingual_db_1 {db} { $db eval { INSERT INTO t2(docid, x, y, l) VALUES($i, $x, $y, $iLangid) } } + + $db eval { + CREATE TABLE data(x, y, l); + INSERT INTO data(rowid, x, y, l) SELECT docid, x, y, l FROM t2; + } } proc rowid_list_set_langid {langid} { @@ -159,9 +164,9 @@ proc rowid_list_set_langid {langid} { proc rowid_list {pattern} { set langid $::rowid_list_langid set res [list] - db eval {SELECT docid, x, y FROM t2 WHERE l = $langid ORDER BY docid ASC} { + db eval {SELECT rowid, x, y FROM data WHERE l = $langid ORDER BY rowid ASC} { if {[string match "*$pattern*" $x] || [string match "*$pattern*" $y]} { - lappend res $docid + lappend res $rowid } } return $res @@ -235,7 +240,7 @@ do_test 2.0 { build_multilingual_db_1 db } {} -proc do_test_2 {tn query res_script} { +proc do_test_query1 {tn query res_script} { for {set langid 0} {$langid < 10} {incr langid} { rowid_list_set_langid $langid set res [eval $res_script] @@ -243,17 +248,17 @@ proc do_test_2 {tn query res_script} { set actual [ execsql {SELECT docid FROM t2 WHERE t2 MATCH $query AND l = $langid} ] - do_test 2.$tn.$langid [list set {} $actual] $res + do_test $tn.$langid [list set {} $actual] $res } } # Run some queries. -do_test_2 1.1 {delta} { rowid_list delta } -do_test_2 1.2 {"zero one two"} { rowid_list "zero one two" } -do_test_2 1.3 {zero one two} { +do_test_query1 2.1.1 {delta} { rowid_list delta } +do_test_query1 2.1.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 2.1.3 {zero one two} { and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] } -do_test_2 1.4 {"zero one" OR "one two"} { +do_test_query1 2.1.4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } @@ -264,12 +269,12 @@ do_execsql_test 2.2 { INSERT INTO t2(t2) VALUES('optimize'); SELECT count(*) FROM t2_segdir; } {9} -do_test_2 2.1 {delta} { rowid_list delta } -do_test_2 2.2 {"zero one two"} { rowid_list "zero one two" } -do_test_2 2.3 {zero one two} { +do_test_query1 2.2.1 {delta} { rowid_list delta } +do_test_query1 2.2.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 2.2.3 {zero one two} { and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] } -do_test_2 2.4 {"zero one" OR "one two"} { +do_test_query1 2.2.4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } @@ -280,14 +285,64 @@ do_test 2.3 { build_multilingual_db_1 db execsql { INSERT INTO t2(t2) VALUES('rebuild') } } {} -do_test_2 3.1 {delta} { rowid_list delta } -do_test_2 3.2 {"zero one two"} { rowid_list "zero one two" } -do_test_2 3.3 {zero one two} { +do_test_query1 2.3.1 {delta} { rowid_list delta } +do_test_query1 2.3.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 2.3.3 {zero one two} { and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] } -do_test_2 3.4 {"zero one" OR "one two"} { +do_test_query1 2.3.4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } +#------------------------------------------------------------------------- +# Test cases 3.* +# +do_test 3.0 { + reset_db + build_multilingual_db_1 db + execsql { + CREATE TABLE t3_data(l, x, y); + INSERT INTO t3_data(rowid, l, x, y) SELECT docid, l, x, y FROM t2; + DROP TABLE t2; + } +} {} +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t2 USING fts4(content=t3_data, languageid=l); + INSERT INTO t2(t2) VALUES('rebuild'); +} + +do_test_query1 3.1.1 {delta} { rowid_list delta } +do_test_query1 3.1.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 3.1.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_query1 3.1.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + +do_execsql_test 3.2.1 { + DROP TABLE t2; + CREATE VIRTUAL TABLE t2 USING fts4(x, y, languageid=l, content=nosuchtable); +} + +do_execsql_test 3.2.2 { + INSERT INTO t2(docid, x, y, l) SELECT rowid, x, y, l FROM t3_data; +} + +do_execsql_test 3.2.3 { + DROP TABLE t3_data; +} + +do_test_query1 3.3.1 {delta} { rowid_list delta } +do_test_query1 3.3.2 {"zero one two"} { rowid_list "zero one two" } +do_test_query1 3.3.3 {zero one two} { + and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] +} +do_test_query1 3.3.4 {"zero one" OR "one two"} { + or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] +} + + + finish_test From 9db1d56f02e3280b3b8f2cba8e392ad29cbb8865 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 Mar 2012 20:34:57 +0000 Subject: [PATCH 47/51] Remove an obsolete comment from FTS3. No code changes. FossilOrigin-Name: 74d224b14467d8a6915d3cf372f45e0d28708fa2 --- ext/fts3/fts3.c | 4 ---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index d17809fe54..1e8689dc0e 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -286,10 +286,6 @@ ** will eventually overtake the earlier data and knock it out. The ** query logic likewise merges doclists so that newer data knocks out ** older data. -** -** TODO(shess) Provide a VACUUM type operation to clear out all -** deletions and duplications. This would basically be a forced merge -** into a single segment. */ #include "fts3Int.h" diff --git a/manifest b/manifest index 1bfda8c87a..1ecac2891d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\sheader\scomment\sof\ssqlite3_commit_hook(). -D 2012-03-01T21:30:44.361 +C Remove\san\sobsolete\scomment\sfrom\sFTS3.\s\sNo\scode\schanges. +D 2012-03-02T20:34:57.831 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,7 +63,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 4cf7b8e5bbb6667f5d7818fa0bf064fbbb72b086 +F ext/fts3/fts3.c 96ae9e273a770e2c249117360ec68e38aa0b13fa F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h ce958a6fa92a95462853aa3acc0b69bcda39102f F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 @@ -991,7 +991,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 16330a2f7262173a32ae48a72c0ee2522b6dc554 -R 4fca0028f2954d371f8ed29b9be73d18 +P 3784d1475b49f24e4a51450edc94aee66fd48fe4 +R 8b20c4d4f44baa368677e57454570c9f U drh -Z a58e0a8dbfee9129245f4f54328583b6 +Z 21e716094cd8c8ff2e7fa8079fce4100 diff --git a/manifest.uuid b/manifest.uuid index 7506854a4c..056c76d316 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3784d1475b49f24e4a51450edc94aee66fd48fe4 \ No newline at end of file +74d224b14467d8a6915d3cf372f45e0d28708fa2 \ No newline at end of file From 5b044545d703500c4487bfc74165ca318bece20e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 2 Mar 2012 22:41:06 +0000 Subject: [PATCH 48/51] Fix test pragma-19.5 so that it works on file names that may contain spaces and/or backslashes. FossilOrigin-Name: 9aaa1ab7c63c9045fecf5e965b333cc5f28b0791 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/pragma.test | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1ecac2891d..6fa62fc177 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sobsolete\scomment\sfrom\sFTS3.\s\sNo\scode\schanges. -D 2012-03-02T20:34:57.831 +C Fix\stest\spragma-19.5\sso\sthat\sit\sworks\son\sfile\snames\sthat\smay\scontain\sspaces\sand/or\sbackslashes. +D 2012-03-02T22:41:06.784 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -635,7 +635,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/permutations.test fa6f0e5f13fe0b1d3f7a7613179b7f7b20028184 -F test/pragma.test f6111ded4d56b79436a60a757d62f3c96a9cf3f5 +F test/pragma.test f11c59ec935a52edb4d3d5676d456588121fcefa F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 @@ -991,7 +991,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 3784d1475b49f24e4a51450edc94aee66fd48fe4 -R 8b20c4d4f44baa368677e57454570c9f -U drh -Z 21e716094cd8c8ff2e7fa8079fce4100 +P 74d224b14467d8a6915d3cf372f45e0d28708fa2 +R 1b3b0133ab927bace43b063fe5fb3563 +U mistachkin +Z 8931ed203e5c3d9dfd3529ebd0c654df diff --git a/manifest.uuid b/manifest.uuid index 056c76d316..e9e40ff7c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -74d224b14467d8a6915d3cf372f45e0d28708fa2 \ No newline at end of file +9aaa1ab7c63c9045fecf5e965b333cc5f28b0791 \ No newline at end of file diff --git a/test/pragma.test b/test/pragma.test index 00be9b8f6a..3f17bd742b 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1507,7 +1507,7 @@ do_test pragma-19.4 { catchsql {PRAGMA error=7} } {1 {out of memory}} do_test pragma-19.5 { - file tail [execsql {PRAGMA filename}] + file tail [lindex [execsql {PRAGMA filename}] 0] } {test.db} From 7f67d94cfe8eb49e62cf6766bc409b6831f4634c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 Mar 2012 00:34:47 +0000 Subject: [PATCH 49/51] Fix a bug [3557ad65a07] that causes incorrect DISTINCT processing on an indexed query involving the IN operator. FossilOrigin-Name: dec9a995d7012a1f85be4edb71240b9d4f07c60e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6fa62fc177..3e364d2766 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stest\spragma-19.5\sso\sthat\sit\sworks\son\sfile\snames\sthat\smay\scontain\sspaces\sand/or\sbackslashes. -D 2012-03-02T22:41:06.784 +C Fix\sa\sbug\s[3557ad65a07]\sthat\scauses\sincorrect\sDISTINCT\sprocessing\son\nan\sindexed\squery\sinvolving\sthe\sIN\soperator. +D 2012-03-03T00:34:47.462 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -253,7 +253,7 @@ F src/vtab.c ab90fb600a3f5e4b7c48d22a4cdb2d6b23239847 F src/wal.c 7bb3ad807afc7973406c805d5157ec7a2f65e146 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c af623942514571895818b9b7ae11db95ae3b3d88 +F src/where.c f2cf59751f7facb4c422adf83ddc989aa5772874 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -991,7 +991,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 74d224b14467d8a6915d3cf372f45e0d28708fa2 -R 1b3b0133ab927bace43b063fe5fb3563 -U mistachkin -Z 8931ed203e5c3d9dfd3529ebd0c654df +P 9aaa1ab7c63c9045fecf5e965b333cc5f28b0791 +R 0fde9a1f5b219b6e9128e202fe652a01 +U drh +Z 8bae0564fba3f81fa4a316b685f23026 diff --git a/manifest.uuid b/manifest.uuid index e9e40ff7c1..41df7bd95d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9aaa1ab7c63c9045fecf5e965b333cc5f28b0791 \ No newline at end of file +dec9a995d7012a1f85be4edb71240b9d4f07c60e \ No newline at end of file diff --git a/src/where.c b/src/where.c index 78efbf9790..eda8d5fb8e 100644 --- a/src/where.c +++ b/src/where.c @@ -3103,7 +3103,9 @@ static void bestBtreeIndex( /* If there is a DISTINCT qualifier and this index will scan rows in ** order of the DISTINCT expressions, clear bDist and set the appropriate ** flags in wsFlags. */ - if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){ + if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) + && (wsFlags & WHERE_COLUMN_IN)==0 + ){ bDist = 0; wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } From c724008b2669e0c46c60af82a8c692d244ac9fd7 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 Mar 2012 01:44:12 +0000 Subject: [PATCH 50/51] Add a test case for ticket [3557ad65a076c]. FossilOrigin-Name: 4f34d7077b9acf7926c5e7375ca870d4ed3c60b9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/select4.test | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3e364d2766..3d02667f89 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\s[3557ad65a07]\sthat\scauses\sincorrect\sDISTINCT\sprocessing\son\nan\sindexed\squery\sinvolving\sthe\sIN\soperator. -D 2012-03-03T00:34:47.462 +C Add\sa\stest\scase\sfor\sticket\s[3557ad65a076c]. +D 2012-03-03T01:44:12.761 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -669,7 +669,7 @@ F test/securedel.test 87a2561151af1f1e349071a89fdd77059f50113c F test/select1.test deba017eed9daa5af33de868676c997e7eebb931 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054 -F test/select4.test 44aa6e7110592e18110b0b9cf5c024d37d23be17 +F test/select4.test 00179be44e531fe04c1c3f15df216439dff2519d F test/select5.test e758b8ef94f69b111df4cb819008856655dcd535 F test/select6.test cc25a8650cf9a4d4f74e586c45a75f9836516b18 F test/select7.test dad6f00f0d49728a879d6eb6451d4752db0b0abe @@ -991,7 +991,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 9aaa1ab7c63c9045fecf5e965b333cc5f28b0791 -R 0fde9a1f5b219b6e9128e202fe652a01 +P dec9a995d7012a1f85be4edb71240b9d4f07c60e +R b4cefaed66b163e0022c525667456b34 U drh -Z 8bae0564fba3f81fa4a316b685f23026 +Z 8282af4a85e83e5f9dade049cda1072b diff --git a/manifest.uuid b/manifest.uuid index 41df7bd95d..84b5cbe829 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dec9a995d7012a1f85be4edb71240b9d4f07c60e \ No newline at end of file +4f34d7077b9acf7926c5e7375ca870d4ed3c60b9 \ No newline at end of file diff --git a/test/select4.test b/test/select4.test index dff0b90fee..e205b37a4f 100644 --- a/test/select4.test +++ b/test/select4.test @@ -805,4 +805,23 @@ do_test select4-12.1 { } ;# ifcapable compound + +# Ticket [3557ad65a076c] - Incorrect DISTINCT processing with an +# indexed query using IN. +# +do_test select4-13.1 { + sqlite3 db test.db + db eval { + CREATE TABLE t13(a,b); + INSERT INTO t13 VALUES(1,1); + INSERT INTO t13 VALUES(2,1); + INSERT INTO t13 VALUES(3,1); + INSERT INTO t13 VALUES(2,2); + INSERT INTO t13 VALUES(3,2); + INSERT INTO t13 VALUES(4,2); + CREATE INDEX t13ab ON t13(a,b); + SELECT DISTINCT b from t13 WHERE a IN (1,2,3); + } +} {1 2} + finish_test From 996073b3aec31f41354a25148bbc995c1391633c Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 Mar 2012 18:46:41 +0000 Subject: [PATCH 51/51] Add the xLanguageid method to sqlite3_fts3_tokenizer versions 1 and greater. FossilOrigin-Name: f8e9c445dd358c40e5a7bf3756b9f291909dbea7 --- ext/fts3/fts3.c | 10 +- ext/fts3/fts3Int.h | 6 +- ext/fts3/fts3_expr.c | 43 ++++++-- ext/fts3/fts3_snippet.c | 16 +-- ext/fts3/fts3_test.c | 199 ++++++++++++++++++++++++++++++++++++++ ext/fts3/fts3_tokenizer.c | 3 +- ext/fts3/fts3_tokenizer.h | 11 ++- ext/fts3/fts3_write.c | 39 ++++---- manifest | 30 +++--- manifest.uuid | 2 +- test/fts4langid.test | 41 +++++++- test/permutations.test | 3 +- 12 files changed, 340 insertions(+), 63 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index dec7f8722c..f16191f31f 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2958,8 +2958,11 @@ static int fts3FilterMethod( return SQLITE_NOMEM; } - rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat, - p->nColumn, iCol, zQuery, -1, &pCsr->pExpr + pCsr->iLangid = 0; + if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); + + rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, + p->azColumn, p->bHasStat, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ @@ -2969,9 +2972,6 @@ static int fts3FilterMethod( return rc; } - pCsr->iLangid = 0; - if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); - rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 078b5b987b..393cd6aea1 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -498,7 +498,7 @@ void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ -int sqlite3Fts3ExprParse(sqlite3_tokenizer *, +int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, char **, int, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); @@ -507,6 +507,10 @@ int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); int sqlite3Fts3InitTerm(sqlite3 *db); #endif +int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); + /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 1c3a79071c..a6e3492242 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -92,6 +92,7 @@ int sqlite3_fts3_enable_parentheses = 0; typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + int iLangid; /* Language id used with tokenizer */ const char **azCol; /* Array of column names for fts3 table */ int bFts4; /* True to allow FTS4-only syntax */ int nCol; /* Number of entries in azCol[] */ @@ -127,6 +128,33 @@ static void *fts3MallocZero(int nByte){ return pRet; } +int sqlite3Fts3OpenTokenizer( + sqlite3_tokenizer *pTokenizer, + int iLangid, + const char *z, + int n, + sqlite3_tokenizer_cursor **ppCsr +){ + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr = 0; + int rc; + + rc = pModule->xOpen(pTokenizer, z, n, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( rc==SQLITE_OK ){ + pCsr->pTokenizer = pTokenizer; + if( pModule->iVersion>=1 ){ + rc = pModule->xLanguageid(pCsr, iLangid); + if( rc!=SQLITE_OK ){ + pModule->xClose(pCsr); + pCsr = 0; + } + } + } + *ppCsr = pCsr; + return rc; +} + /* ** Extract the next token from buffer z (length n) using the tokenizer @@ -154,15 +182,13 @@ static int getNextToken( Fts3Expr *pRet = 0; int nConsumed = 0; - rc = pModule->xOpen(pTokenizer, z, n, &pCursor); + rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; int nToken, iStart, iEnd, iPosition; int nByte; /* total space to allocate */ - pCursor->pTokenizer = pTokenizer; rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); - if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; pRet = (Fts3Expr *)fts3MallocZero(nByte); @@ -268,10 +294,10 @@ static int getNextString( ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase ** structures. */ - rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); + rc = sqlite3Fts3OpenTokenizer( + pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); if( rc==SQLITE_OK ){ int ii; - pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ const char *zByte; int nByte, iBegin, iEnd, iPos; @@ -745,6 +771,7 @@ exprparse_out: */ int sqlite3Fts3ExprParse( sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ char **azCol, /* Array of column names for fts3 table */ int bFts4, /* True to allow FTS4-only syntax */ int nCol, /* Number of entries in azCol[] */ @@ -755,11 +782,13 @@ int sqlite3Fts3ExprParse( int nParsed; int rc; ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; sParse.azCol = (const char **)azCol; sParse.nCol = nCol; sParse.iDefaultCol = iDefaultCol; - sParse.nNest = 0; sParse.bFts4 = bFts4; if( z==0 ){ *ppExpr = 0; @@ -950,7 +979,7 @@ static void fts3ExprTest( } rc = sqlite3Fts3ExprParse( - pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr ); if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ sqlite3_result_error(context, "Error parsing expression", -1); diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 23ef25c5d4..fd5bc9786b 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -532,6 +532,7 @@ static int fts3StringAppend( */ static int fts3SnippetShift( Fts3Table *pTab, /* FTS3 table snippet comes from */ + int iLangid, /* Language id to use in tokenizing */ int nSnippet, /* Number of tokens desired for snippet */ const char *zDoc, /* Document text to extract snippet from */ int nDoc, /* Size of buffer zDoc in bytes */ @@ -567,11 +568,10 @@ static int fts3SnippetShift( /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) ** or more tokens in zDoc/nDoc. */ - rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); if( rc!=SQLITE_OK ){ return rc; } - pC->pTokenizer = pTab->pTokenizer; while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3; rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); @@ -631,11 +631,10 @@ static int fts3SnippetText( /* Open a token cursor on the document. */ pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; - rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); if( rc!=SQLITE_OK ){ return rc; } - pC->pTokenizer = pTab->pTokenizer; while( rc==SQLITE_OK ){ int iBegin; /* Offset in zDoc of start of token */ @@ -657,7 +656,9 @@ static int fts3SnippetText( if( !isShiftDone ){ int n = nDoc - iBegin; - rc = fts3SnippetShift(pTab, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask); + rc = fts3SnippetShift( + pTab, pCsr->iLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask + ); isShiftDone = 1; /* Now that the shift has been done, check if the initial "..." are @@ -1390,9 +1391,10 @@ void sqlite3Fts3Offsets( } /* Initialize a tokenizer iterator to iterate through column iCol. */ - rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, + zDoc, nDoc, &pC + ); if( rc!=SQLITE_OK ) goto offsets_out; - pC->pTokenizer = pTab->pTokenizer; rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); while( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 72735f3d12..7cbc9eae9f 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -13,6 +13,9 @@ ** This file is not part of the production FTS code. It is only used for ** testing. It contains a Tcl command that can be used to test if a document ** matches an FTS NEAR expression. +** +** As of March 2012, it also contains a version 1 tokenizer used for testing +** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly. */ #include @@ -314,11 +317,207 @@ static int fts3_configure_incr_load_cmd( return TCL_OK; } +/************************************************************************** +** Beginning of test tokenizer code. +** +** For language 0, this tokenizer is similar to the default 'simple' +** tokenizer. For other languages L, the following: +** +** * Odd numbered languages are case-sensitive. Even numbered +** languages are not. +** +** * Language ids 100 or greater are considered an error. +** +** The implementation assumes that the input contains only ASCII characters +** (i.e. those that may be encoded in UTF-8 using a single byte). +*/ +typedef struct test_tokenizer { + sqlite3_tokenizer base; +} test_tokenizer; + +typedef struct test_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *aInput; /* Input being tokenized */ + int nInput; /* Size of the input in bytes */ + int iInput; /* Current offset in aInput */ + int iToken; /* Index of next token to be returned */ + char *aBuffer; /* Buffer containing current token */ + int nBuffer; /* Number of bytes allocated at pToken */ + int iLangid; /* Configured language id */ +} test_tokenizer_cursor; + +static int testTokenizerCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + test_tokenizer *pNew; + + pNew = sqlite3_malloc(sizeof(test_tokenizer)); + if( !pNew ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(test_tokenizer)); + + *ppTokenizer = (sqlite3_tokenizer *)pNew; + return SQLITE_OK; +} + +static int testTokenizerDestroy(sqlite3_tokenizer *pTokenizer){ + test_tokenizer *p = (test_tokenizer *)pTokenizer; + sqlite3_free(p); + return SQLITE_OK; +} + +static int testTokenizerOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *pInput, int nBytes, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + int rc = SQLITE_OK; /* Return code */ + test_tokenizer_cursor *pCsr; /* New cursor object */ + + UNUSED_PARAMETER(pTokenizer); + + pCsr = (test_tokenizer_cursor *)sqlite3_malloc(sizeof(test_tokenizer_cursor)); + if( pCsr==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(test_tokenizer_cursor)); + pCsr->aInput = pInput; + if( nBytes<0 ){ + pCsr->nInput = strlen(pInput); + }else{ + pCsr->nInput = nBytes; + } + } + + *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; + return rc; +} + +static int testTokenizerClose(sqlite3_tokenizer_cursor *pCursor){ + test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; + sqlite3_free(pCsr->aBuffer); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int testIsTokenChar(char c){ + return (c>='a' && c<='z') || (c>='A' && c<='Z'); +} +static int testTolower(char c){ + char ret = c; + if( ret>='A' && ret<='Z') ret = ret - ('A'-'a'); + return ret; +} + +static int testTokenizerNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by testTokenizerOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; + int rc = SQLITE_OK; + const char *p; + const char *pEnd; + + p = &pCsr->aInput[pCsr->iInput]; + pEnd = &pCsr->aInput[pCsr->nInput]; + + /* Skip past any white-space */ + assert( p<=pEnd ); + while( ppCsr->nBuffer ){ + sqlite3_free(pCsr->aBuffer); + pCsr->aBuffer = sqlite3_malloc(nToken); + } + if( pCsr->aBuffer==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + + if( pCsr->iLangid & 0x00000001 ){ + for(i=0; iaBuffer[i] = pToken[i]; + }else{ + for(i=0; iaBuffer[i] = testTolower(pToken[i]); + } + pCsr->iToken++; + pCsr->iInput = p - pCsr->aInput; + + *ppToken = pCsr->aBuffer; + *pnBytes = nToken; + *piStartOffset = pToken - pCsr->aInput; + *piEndOffset = p - pCsr->aInput; + *piPosition = pCsr->iToken; + } + } + + return rc; +} + +static int testTokenizerLanguage( + sqlite3_tokenizer_cursor *pCursor, + int iLangid +){ + int rc = SQLITE_OK; + test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; + pCsr->iLangid = iLangid; + if( pCsr->iLangid>=100 ){ + rc = SQLITE_ERROR; + } + return rc; +} + +static int fts3_test_tokenizer_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static const sqlite3_tokenizer_module testTokenizerModule = { + 1, + testTokenizerCreate, + testTokenizerDestroy, + testTokenizerOpen, + testTokenizerClose, + testTokenizerNext, + testTokenizerLanguage + }; + const sqlite3_tokenizer_module *pPtr = &testTokenizerModule; + if( objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj( + (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *) + )); + return TCL_OK; +} + +/* +** End of tokenizer code. +**************************************************************************/ + int Sqlitetestfts3_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); Tcl_CreateObjCommand(interp, "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0 ); + Tcl_CreateObjCommand( + interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0 + ); return TCL_OK; } #endif /* ifdef SQLITE_TEST */ diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index 6494bb96d8..f6b044ff6a 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -288,11 +288,10 @@ static void testFunc( goto finish; } pTokenizer->pModule = p; - if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){ + if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ zErr = "error in xOpen()"; goto finish; } - pCsr->pTokenizer = pTokenizer; while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); diff --git a/ext/fts3/fts3_tokenizer.h b/ext/fts3/fts3_tokenizer.h index 615644506c..c91c7ed790 100644 --- a/ext/fts3/fts3_tokenizer.h +++ b/ext/fts3/fts3_tokenizer.h @@ -52,7 +52,7 @@ typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; struct sqlite3_tokenizer_module { /* - ** Structure version. Should always be set to 0. + ** Structure version. Should always be set to 0 or 1. */ int iVersion; @@ -133,6 +133,15 @@ struct sqlite3_tokenizer_module { int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ int *piPosition /* OUT: Number of tokens returned before this one */ ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); }; struct sqlite3_tokenizer { diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index decbe0b73a..8f97c8be98 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -657,6 +657,7 @@ static int fts3PendingTermsAddOne( */ static int fts3PendingTermsAdd( Fts3Table *p, /* Table into which text will be inserted */ + int iLangid, /* Language id to use */ const char *zText, /* Text of document to be inserted */ int iCol, /* Column into which text is being inserted */ u32 *pnWord /* OUT: Number of tokens inserted */ @@ -686,11 +687,10 @@ static int fts3PendingTermsAdd( return SQLITE_OK; } - rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr); + rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); if( rc!=SQLITE_OK ){ return rc; } - pCsr->pTokenizer = pTokenizer; xNext = pModule->xNext; while( SQLITE_OK==rc @@ -783,11 +783,16 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ ** Argument apVal is the same as the similarly named argument passed to ** fts3InsertData(). Parameter iDocid is the docid of the new row. */ -static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ +static int fts3InsertTerms( + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, + u32 *aSz +){ int i; /* Iterator variable */ for(i=2; inColumn+2; i++){ const char *zText = (const char *)sqlite3_value_text(apVal[i]); - int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, i-2, &aSz[i-2]); if( rc!=SQLITE_OK ){ return rc; } @@ -933,13 +938,11 @@ static void fts3DeleteTerms( if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; - rc = fts3PendingTermsDocid(p, - langidFromSelect(p, pSelect), - sqlite3_column_int64(pSelect, 0) - ); + int iLangid = langidFromSelect(p, pSelect); + rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0)); for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ const char *zText = (const char *)sqlite3_column_text(pSelect, i); - rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]); aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); } if( rc!=SQLITE_OK ){ @@ -3102,13 +3105,12 @@ static int fts3DoRebuild(Fts3Table *p){ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ int iCol; - rc = fts3PendingTermsDocid(p, - langidFromSelect(p, pStmt), sqlite3_column_int64(pStmt, 0) - ); + int iLangid = langidFromSelect(p, pStmt); + rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); aSz[p->nColumn] = 0; for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); - rc = fts3PendingTermsAdd(p, z, iCol, &aSz[iCol]); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); } if( p->bHasDocsize ){ @@ -3227,14 +3229,13 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); sqlite3_tokenizer_cursor *pTC = 0; - rc = pModule->xOpen(pT, zText, -1, &pTC); + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); while( rc==SQLITE_OK ){ char const *zToken; /* Buffer containing token */ int nToken; /* Number of bytes in token */ int iDum1, iDum2; /* Dummy variables */ int iPos; /* Position of token in zText */ - pTC->pTokenizer = pT; rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ Fts3PhraseToken *pPT = pDef->pToken; @@ -3467,6 +3468,7 @@ int sqlite3Fts3UpdateMethod( /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ + int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); if( bInsertDone==0 ){ rc = fts3InsertData(p, apVal, pRowid); if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ @@ -3474,14 +3476,11 @@ int sqlite3Fts3UpdateMethod( } } if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ - rc = fts3PendingTermsDocid(p, - sqlite3_value_int(apVal[2 + p->nColumn + 2]), - *pRowid - ); + rc = fts3PendingTermsDocid(p, iLangid, *pRowid); } if( rc==SQLITE_OK ){ assert( p->iPrevDocid==*pRowid ); - rc = fts3InsertTerms(p, apVal, aSzIns); + rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); } if( p->bHasDocsize ){ fts3InsertDocsize(&rc, p, aSzIns); diff --git a/manifest b/manifest index 15fcd20c9b..ef077935d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\swith\scombining\scontent=\sand\slanguageid=\sin\sa\ssingle\sfts4\stable. -D 2012-03-02T19:53:02.350 +C Add\sthe\sxLanguageid\smethod\sto\ssqlite3_fts3_tokenizer\sversions\s1\sand\sgreater. +D 2012-03-03T18:46:41.456 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -63,22 +63,22 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c fd89caa4169520c32cf46ca5a62df6dd48201422 +F ext/fts3/fts3.c fcda9a9ff7ccfb9fe4388d36063e3405a652e15f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 521d300f2af4e741f53c4e2dd540275fb64533eb +F ext/fts3/fts3Int.h d1d7f964ddee067bcd16a6af4ba7ecf66220056d F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239 -F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15 +F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa F ext/fts3/fts3_porter.c b7e5276f9f0a5fc7018b6fa55ce0f31f269ef881 -F ext/fts3/fts3_snippet.c 1f9ee6a8e0e242649645968dcec4deb253d86c2a +F ext/fts3/fts3_snippet.c c9e126c20760988aa7c43c6ea1379db34738282e F ext/fts3/fts3_term.c d3466cf99432291be08e379d89645462431809d6 -F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 -F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 -F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 +F ext/fts3/fts3_test.c a026412a41450a014ccb7abdd5efaa7c9711d49e +F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce +F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 35b98a42f9bbdd28af1b1f3bb0c09ff07090a764 +F ext/fts3/fts3_write.c f87bb2d27d31cb7a7bf306747079095393c9d073 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f -F test/fts4langid.test be989b5cddcd7596b87232af193f6c4560a34272 +F test/fts4langid.test fabdd5a8db0fa00292e0704809f566e3fb6dba3a F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -635,7 +635,7 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test fa6f0e5f13fe0b1d3f7a7613179b7f7b20028184 +F test/permutations.test 2b5a1b64a8e5114757457fbce9010387d1fe7682 F test/pragma.test f6111ded4d56b79436a60a757d62f3c96a9cf3f5 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 181bc35731f19c3e4497ba3338c209918d34ea69 -R acfb20f690a18ac8d67e116ae8c76f7d +P 22491e7bc38aee43819b888e04241cb6a6ef73a3 +R b3e8f4cc7fcd5d7bfa4510a8d2434f16 U dan -Z b771fbbc3f5dd49e39970a5f917fc8b7 +Z 8fb02bd9c9b9481fcea2986d10daaba1 diff --git a/manifest.uuid b/manifest.uuid index def5beae28..868a4cc932 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -22491e7bc38aee43819b888e04241cb6a6ef73a3 \ No newline at end of file +f8e9c445dd358c40e5a7bf3756b9f291909dbea7 \ No newline at end of file diff --git a/test/fts4langid.test b/test/fts4langid.test index b9110cb768..08f1a21c17 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -40,7 +40,6 @@ set ::testprefix fts4langid # 3.* - Tests with content= tables. Both where there is a real # underlying content table and where there is not. # -# # 4.* - Test that if one is provided, the tokenizer xLanguage method # is called to configure the tokenizer before tokenizing query # or document text. @@ -342,7 +341,45 @@ do_test_query1 3.3.4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } +#------------------------------------------------------------------------- +# Test cases 4.* +# +proc build_multilingual_db_2 {db} { + $db eval { + CREATE VIRTUAL TABLE t4 USING fts4( + tokenize=testtokenizer, + languageid=lid + ); + } + for {set i 0} {$i < 50} {incr i} { + execsql { + INSERT INTO t4(docid, content, lid) VALUES($i, 'The Quick Brown Fox', $i) + } + } +} +do_test 4.1.0 { + reset_db + set ptr [fts3_test_tokenizer] + execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) } + build_multilingual_db_2 db +} {} +do_execsql_test 4.1.1 { + SELECT docid FROM t4 WHERE t4 MATCH 'quick'; +} {0} +do_execsql_test 4.1.2 { + SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1; +} {} +do_execsql_test 4.1.3 { + SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1; +} {1} +for {set i 0} {$i < 50} {incr i} { + do_execsql_test 4.1.4.$i { + SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i; + } [expr 0==($i%2)] +} +do_catchsql_test 4.1.5 { + INSERT INTO t4(content, lid) VALUES('hello world', 101) +} {1 {SQL logic error or missing database}} finish_test - diff --git a/test/permutations.test b/test/permutations.test index f55db1be73..26c1b2a514 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -184,8 +184,7 @@ test_suite "fts3" -prefix "" -description { fts3aux1.test fts3comp1.test fts3auto.test fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test - fts3corrupt2.test - fts3first.test + fts3corrupt2.test fts3first.test fts4langid.test }