From 4edc6bf3eed92fc5a5547406dec6760c9b062fcf Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 10 May 2011 17:31:29 +0000 Subject: [PATCH 01/69] Add experimental support for read-only connections to WAL databases. FossilOrigin-Name: bb59f9862da45d25fb51d7821130854828c91c98 --- manifest | 26 ++++++---- manifest.uuid | 2 +- src/os_unix.c | 51 +++++++++++++----- src/sqlite.h.in | 4 ++ src/test1.c | 3 ++ src/wal.c | 59 +++++++++++++++------ test/lock_common.tcl | 4 +- test/walro.test | 121 +++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 227 insertions(+), 43 deletions(-) create mode 100644 test/walro.test diff --git a/manifest b/manifest index daee9754e7..50889411e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Return\sa\ssuitable\serror\smessage\sif\sthe\smode=\sargument\sto\sa\sURI\sspecifies\na\shigher\smode\sthan\swhat\sis\sallowed\sby\scontext.\s\sOther\sminor\scleanups\sfor\nthe\sURI\sparsing\slogic. -D 2011-05-09T19:20:17.775 +C Add\sexperimental\ssupport\sfor\sread-only\sconnections\sto\sWAL\sdatabases. +D 2011-05-10T17:31:29.338 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7a4d9524721d40ef9ee26f93f9bd6a51dba106f2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/os.c 22ac61d06e72a0dac900400147333b07b13d8e1d F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 2c67d126874b78eb427371db4793f0e8fbc7448b +F src/os_unix.c 03630dd062c3d1fb9f25e2a227048b709c5babff F src/os_win.c ff0e14615a5086fa5ba6926e4ec0dc9cfb5a1a84 F src/pager.c 24b689bc3639d534f5fb292d2c68038b1e720527 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 @@ -179,14 +179,14 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff F src/shell.c 72e7e176bf46d5c6518d15ac4ad6847c4bb5df79 -F src/sqlite.h.in 41a0e4bc842917226e170273f64b95717a63270a +F src/sqlite.h.in e7bbcb330ced6b5e25c9db8089c2c77aaefadf7d F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 F src/sqliteInt.h b34bd64a7ade4808fcc301e0bb67ef5051ea49c6 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 501c9a200fd998a268be475be5858febc90b725b -F src/test1.c f506164085bc4cbbb4b5c14b9c6de153f1aeafc5 +F src/test1.c 6ae026cd9d2b1b1e95a372a7460d091705db645d F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 @@ -245,7 +245,7 @@ F src/vdbeblob.c c3ccb7c8732858c680f442932e66ad06bb036562 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 48dcef8bc757c2e7b488f68b5ddebb1650da2450 -F src/wal.c 7334009b396285b658a95a3b6bc6d2b016a1f794 +F src/wal.c 2574b16ee88b1934e97f63ae38025dc48ef45327 F src/wal.h 7a5fbb00114b7f2cd40c7e1003d4c41ce9d26840 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 @@ -547,7 +547,7 @@ F test/lock4.test c82268c031d39345d05efa672f80b025481b3ae5 F test/lock5.test b2abb5e711bc59b0eae00f6c97a36ec9f458fada F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 64006c84c1c616657e237c7ad6532b765611cf64 -F test/lock_common.tcl d279887a0ab16cdb6d935c1203e64113c5a000e9 +F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 9d7bbfcc1b52c88ba7b2ba6554068ecf9939f252 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 @@ -883,6 +883,7 @@ F test/walfault.test 58fce626359c9376fe35101b5c0f2df8040aa839 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 22ddccd073c817ac9ead62b88ac446e8dedc7d2c F test/walnoshm.test a074428046408f4eb5c6a00e09df8cc97ff93317 +F test/walro.test 0fb4c79a9dfa1d14f46859e469d3b4844480cd9d F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933 F test/walthread.test a25a393c068a2b42b44333fa3fdaae9072f1617c @@ -935,7 +936,10 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P ca3797d4967361e31a8a5ce1ce8190b095f3ed4c -R fbfaa47316ae48b91e92f26cbe2f2742 -U drh -Z 82e3dc394cc5052bfb9a5a6a21e4f298 +P d9bc1c7fe0ca5f6973a85827330958f4d09f8171 +R 864e9c4ee7a6cf5f8240415fb38d1af8 +T *branch * wal-readonly +T *sym-wal-readonly * +T -sym-trunk * +U dan +Z 321776a9f8d852e48652ad186c3a479b diff --git a/manifest.uuid b/manifest.uuid index 2b8787f1e2..e39ce5418f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9bc1c7fe0ca5f6973a85827330958f4d09f8171 \ No newline at end of file +bb59f9862da45d25fb51d7821130854828c91c98 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index a760e2c147..d463e5917f 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -212,6 +212,7 @@ struct unixFile { UnixUnusedFd *pUnused; /* Pre-allocated UnixUnusedFd */ const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ + int readOnlyShm; /* True to open shared-memory read-only */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ #if SQLITE_ENABLE_LOCKING_STYLE int openFlags; /* The flags specified at open() */ @@ -3452,6 +3453,10 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ case SQLITE_FCNTL_SIZE_HINT: { return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); } + case SQLITE_FCNTL_READONLY_SHM: { + ((unixFile*)id)->readOnlyShm = (pArg!=0); + return SQLITE_OK; + } #ifndef NDEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and @@ -3541,6 +3546,7 @@ struct unixShmNode { char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ + u8 readOnly; /* True if this is a read-only mapping */ #ifdef SQLITE_DEBUG u8 exclMask; /* Mask of exclusive locks held */ u8 sharedMask; /* Mask of shared locks held */ @@ -3780,29 +3786,48 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ - pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, - (sStat.st_mode & 0777)); + int flags = (pDbFd->readOnlyShm ? O_RDONLY : O_RDWR|O_CREAT); + pShmNode->h = robust_open(zShmFilename, flags, (sStat.st_mode & 0777)); if( pShmNode->h<0 ){ rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); goto shm_open_err; } + pShmNode->readOnly = pDbFd->readOnlyShm; /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. - */ - rc = SQLITE_OK; - if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ - if( robust_ftruncate(pShmNode->h, 0) ){ - rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); + ** If not, zero the first few bytes of the shared-memory file to make + ** sure it is not mistaken for valid by code in wal.c. Except, if this + ** is a read-only connection to the shared-memory then it is not possible + ** to check check if another process is holding a read-lock on the DMS + ** byte, as we cannot attempt a write-lock via a read-only file + ** descriptor. So in this case, we just assume the shared-memory + ** contents are Ok and proceed. */ + if( pShmNode->readOnly==0 ){ + rc = SQLITE_OK; + if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ + if( pDbFd->readOnlyShm ){ + rc = SQLITE_IOERR_SHMOPEN; + }else if( 4!=osWrite(pShmNode->h, "\00\00\00\00", 4) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); + } } + if( rc==SQLITE_OK ){ + rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1); + } + if( rc ) goto shm_open_err; } - if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; } } + /* If the unixShmNode is read-only, but SQLITE_FCNTL_READONLY_SHM has not + ** been set for file-descriptor pDbFd, return an error. The wal.c module + ** will then call this function again with SQLITE_FCNTL_READONLY_SHM set. + */ + else if( pShmNode->readOnly && !pDbFd->readOnlyShm ){ + rc = SQLITE_IOERR_SHMOPEN; + goto shm_open_err; + } + /* Make the new connection a child of the unixShmNode */ p->pShmNode = pShmNode; #ifdef SQLITE_DEBUG @@ -3923,7 +3948,7 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE, + pMem = mmap(0, szRegion, PROT_READ|(!pShmNode->readOnly?PROT_WRITE:0), MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion ); if( pMem==MAP_FAILED ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d78daa7672..af88e4e726 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -454,6 +454,9 @@ int sqlite3_exec( #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) + /* ** CAPI3REF: Flags For File Open Operations ** @@ -742,6 +745,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_CHUNK_SIZE 6 #define SQLITE_FCNTL_FILE_POINTER 7 #define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_READONLY_SHM 9 /* diff --git a/src/test1.c b/src/test1.c index 8e30123433..c25fd2289d 100644 --- a/src/test1.c +++ b/src/test1.c @@ -163,6 +163,9 @@ const char *sqlite3TestErrorName(int rc){ case SQLITE_IOERR_CHECKRESERVEDLOCK: zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; default: zName = "SQLITE_Unknown"; break; } return zName; diff --git a/src/wal.c b/src/wal.c index 51ea18fb21..707fe12976 100644 --- a/src/wal.c +++ b/src/wal.c @@ -420,6 +420,7 @@ struct Wal { u8 writeLock; /* True if in a write transaction */ u8 ckptLock; /* True if holding a checkpoint lock */ u8 readOnly; /* True if the WAL file is open read-only */ + u8 readOnlyShm; /* True if the SHM file is open read-only */ WalIndexHdr hdr; /* Wal-index header for current transaction */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ @@ -528,6 +529,16 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); + if( rc==SQLITE_CANTOPEN && iPage==0 ){ + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_READONLY_SHM, (void*)1); + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); + if( rc==SQLITE_OK ){ + pWal->readOnly = pWal->readOnlyShm = 1; + } + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_READONLY_SHM, (void*)0); + } } } @@ -772,6 +783,7 @@ static void walUnlockShared(Wal *pWal, int lockIdx){ } static int walLockExclusive(Wal *pWal, int lockIdx, int n){ int rc; + assert( pWal->readOnlyShm==0 ); if( pWal->exclusiveMode ) return SQLITE_OK; rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); @@ -781,6 +793,7 @@ static int walLockExclusive(Wal *pWal, int lockIdx, int n){ return rc; } static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ + assert( pWal->readOnlyShm==0 ); if( pWal->exclusiveMode ) return; (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); @@ -1056,6 +1069,7 @@ static int walIndexRecover(Wal *pWal){ assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 ); assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); assert( pWal->writeLock ); + assert( pWal->readOnlyShm==0 ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; nLock = SQLITE_SHM_NLOCK - iLock; rc = walLockExclusive(pWal, iLock, nLock); @@ -1904,24 +1918,31 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ badHdr = (page0 ? walIndexTryHdr(pWal, pChanged) : 1); /* If the first attempt failed, it might have been due to a race - ** with a writer. So get a WRITE lock and try again. + ** with a writer. So lock the WAL_WRITE_LOCK byte and try again. */ assert( badHdr==0 || pWal->writeLock==0 ); - if( badHdr && SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ - pWal->writeLock = 1; - if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ - badHdr = walIndexTryHdr(pWal, pChanged); - if( badHdr ){ - /* If the wal-index header is still malformed even while holding - ** a WRITE lock, it can only mean that the header is corrupted and - ** needs to be reconstructed. So run recovery to do exactly that. - */ - rc = walIndexRecover(pWal); - *pChanged = 1; + if( badHdr ){ + if( pWal->readOnlyShm ){ + if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ + walUnlockShared(pWal, WAL_WRITE_LOCK); + rc = SQLITE_READONLY_RECOVERY; } + }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ + pWal->writeLock = 1; + if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ + badHdr = walIndexTryHdr(pWal, pChanged); + if( badHdr ){ + /* If the wal-index header is still malformed even while holding + ** a WRITE lock, it can only mean that the header is corrupted and + ** needs to be reconstructed. So run recovery to do exactly that. + */ + rc = walIndexRecover(pWal); + *pChanged = 1; + } + } + pWal->writeLock = 0; + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } - pWal->writeLock = 0; - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } /* If the header is read successfully, check the version number to make @@ -2108,7 +2129,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } /* There was once an "if" here. The extra "{" is to preserve indentation. */ { - if( mxReadMark < pWal->hdr.mxFrame || mxI==0 ){ + if( pWal->readOnlyShm==0 && (mxReadMark < pWal->hdr.mxFrame || mxI==0) ){ for(i=1; ireadOnlyShm && rc==SQLITE_OK) ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; } rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); @@ -2359,6 +2381,7 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ if( pWal->readOnly ){ return SQLITE_READONLY; } + assert( pWal->readOnlyShm==0 ); /* Only one writer allowed at a time. Get the write lock. Return ** SQLITE_BUSY if unable. @@ -2748,6 +2771,10 @@ int sqlite3WalCheckpoint( assert( pWal->writeLock==0 ); WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + if( pWal->readOnlyShm ){ + return SQLITE_READONLY; + } + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ /* Usually this is SQLITE_BUSY meaning that another thread or process diff --git a/test/lock_common.tcl b/test/lock_common.tcl index 21f584264a..bc1eb86bdc 100644 --- a/test/lock_common.tcl +++ b/test/lock_common.tcl @@ -55,8 +55,8 @@ proc do_multiclient_test {varname script} { uplevel set $varname $tn uplevel $script - code2 { db2 close } - code3 { db3 close } + catch { code2 { db2 close } } + catch { code3 { db3 close } } catch { close $::code2_chan } catch { close $::code3_chan } catch { db close } diff --git a/test/walro.test b/test/walro.test new file mode 100644 index 0000000000..b558622cad --- /dev/null +++ b/test/walro.test @@ -0,0 +1,121 @@ +# 2011 May 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL databases in read-only mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix walro + + +do_multiclient_test tn { + # These tests are only going to work on unix. + # + if {$tcl_platform(platform) != "unix"} continue + + # Do not run tests with the connections in the same process. + # + if {$tn==2} continue + + # Close all connections and delete the database. + # + code1 { db close } + code2 { db2 close } + code3 { db3 close } + forcedelete test.db + forcedelete walro + + file mkdir walro + + do_test 1.1.1 { + code2 { sqlite3 db2 test.db } + sql2 { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('a', 'b'); + } + file exists test.db-shm + } {1} + + do_test 1.1.2 { + file attributes test.db-shm -permissions r--r--r-- + code1 { sqlite3 db test.db } + } {} + + do_test 1.1.3 { sql1 "SELECT * FROM t1" } {a b} + do_test 1.1.4 { sql2 "INSERT INTO t1 VALUES('c', 'd')" } {} + do_test 1.1.5 { sql1 "SELECT * FROM t1" } {a b c d} + + # Check that the read-only connection cannot write or checkpoint the db. + # + do_test 1.1.6 { + csql1 "INSERT INTO t1 VALUES('e', 'f')" + } {1 {attempt to write a readonly database}} + do_test 1.1.7 { + csql1 "PRAGMA wal_checkpoint" + } {1 {attempt to write a readonly database}} + + do_test 1.1.9 { sql2 "INSERT INTO t1 VALUES('e', 'f')" } {} + do_test 1.1.10 { sql1 "SELECT * FROM t1" } {a b c d e f} + + do_test 1.1.11 { + sql2 { + INSERT INTO t1 VALUES('g', 'h'); + PRAGMA wal_checkpoint; + } + set {} {} + } {} + do_test 1.1.12 { sql1 "SELECT * FROM t1" } {a b c d e f g h} + do_test 1.1.13 { sql2 "INSERT INTO t1 VALUES('i', 'j')" } {} + + do_test 1.2.1 { + code2 { db2 close } + code1 { db close } + list [file exists test.db-wal] [file exists test.db-shm] + } {1 1} + do_test 1.2.2 { + code1 { sqlite3 db test.db } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h i j} + + do_test 1.2.3 { + code1 { db close } + file attributes test.db-shm -permissions rw-r--r-- + hexio_write test.db-shm 0 01020304 + file attributes test.db-shm -permissions r--r--r-- + code1 { sqlite3 db test.db } + csql1 { SELECT * FROM t1 } + } {1 {attempt to write a readonly database}} + do_test 1.2.4 { + code1 { sqlite3_extended_errcode db } + } {SQLITE_READONLY_RECOVERY} + + do_test 1.2.5 { + file attributes test.db-shm -permissions rw-r--r-- + code2 { sqlite3 db2 test.db } + sql2 "SELECT * FROM t1" + } {a b c d e f g h i j} + file attributes test.db-shm -permissions r--r--r-- + do_test 1.2.6 { sql1 "SELECT * FROM t1" } {a b c d e f g h i j} + + do_test 1.2.7 { + sql2 { + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES('k', 'l'); + } + set {} {} + } {} + do_test 1.2.8 { sql1 "SELECT * FROM t1" } {a b c d e f g h i j k l} +} + +finish_test From b6d2f9c5ef0d6fa71b744bf5ccfb8ac83b5ed060 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 11 May 2011 14:57:33 +0000 Subject: [PATCH 02/69] Only open a read-only connection to shared-memory if the "readonly_shm=1" option is specified as part of the database file URI (and if a read-write connection fails). FossilOrigin-Name: 671ba5fc59f7a958e5a4138d2425b1173a442ad7 --- manifest | 31 ++++++++++++++----------------- manifest.uuid | 2 +- src/attach.c | 6 ++++-- src/btree.h | 1 + src/main.c | 13 +++++++++++-- src/pager.c | 7 +++++-- src/pager.h | 1 + src/sqliteInt.h | 2 +- src/wal.c | 14 +++++++++++--- src/wal.h | 4 ++-- test/walro.test | 13 ++++++++++--- 11 files changed, 61 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 50889411e1..3d33b8171b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sexperimental\ssupport\sfor\sread-only\sconnections\sto\sWAL\sdatabases. -D 2011-05-10T17:31:29.338 +C Only\sopen\sa\sread-only\sconnection\sto\sshared-memory\sif\sthe\s"readonly_shm=1"\soption\sis\sspecified\sas\spart\sof\sthe\sdatabase\sfile\sURI\s(and\sif\sa\sread-write\sconnection\sfails). +D 2011-05-11T14:57:33.029 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7a4d9524721d40ef9ee26f93f9bd6a51dba106f2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -117,13 +117,13 @@ F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 280f5c04b11b492703a342222b3de0a999445280 F src/analyze.c a425d62e8fa9ebcb4359ab84ff0c62c6563d2e2a -F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f +F src/attach.c a87bfb77a720f7aa02791434aacfd9bc8feb50cc F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 26f8a9d6169413c5682b89b5397d20437b653154 -F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce +F src/btree.h d796dc4030b3cce7f5a0cc4f2f986d2befa6b8ac F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 0132bc6631fa617a1d28ef805921f6dbac18a514 F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a @@ -144,7 +144,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 3ae0d52da013a6326310655be6473fd472347b85 -F src/main.c f00cee5a27f5df5ed536e7043cb451b3c85ce65c +F src/main.c 167fb3285720917e4d1a5c633f558c81b751eca3 F src/malloc.c 591aedb20ae40813f1045f2ef253438a334775d9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 @@ -165,8 +165,8 @@ F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 03630dd062c3d1fb9f25e2a227048b709c5babff F src/os_win.c ff0e14615a5086fa5ba6926e4ec0dc9cfb5a1a84 -F src/pager.c 24b689bc3639d534f5fb292d2c68038b1e720527 -F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 +F src/pager.c 4056376f83f85cea9922a11161087c529e39f7dc +F src/pager.h 34c6b029446f06f40847746d22faac0d354dd909 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 09d38c44ab275db581f7a2f6ff8b9bc7f8c0faaa F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 @@ -181,7 +181,7 @@ F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff F src/shell.c 72e7e176bf46d5c6518d15ac4ad6847c4bb5df79 F src/sqlite.h.in e7bbcb330ced6b5e25c9db8089c2c77aaefadf7d F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h b34bd64a7ade4808fcc301e0bb67ef5051ea49c6 +F src/sqliteInt.h 798fb09648cefc159ac9b3ce5e00f5ada1377ed1 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -245,8 +245,8 @@ F src/vdbeblob.c c3ccb7c8732858c680f442932e66ad06bb036562 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 48dcef8bc757c2e7b488f68b5ddebb1650da2450 -F src/wal.c 2574b16ee88b1934e97f63ae38025dc48ef45327 -F src/wal.h 7a5fbb00114b7f2cd40c7e1003d4c41ce9d26840 +F src/wal.c adc6c83b650e67e32159e11d557a46594834853e +F src/wal.h c1e05cdf3d42ed7c9263739ca8f5cdd8761e7de3 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -883,7 +883,7 @@ F test/walfault.test 58fce626359c9376fe35101b5c0f2df8040aa839 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 22ddccd073c817ac9ead62b88ac446e8dedc7d2c F test/walnoshm.test a074428046408f4eb5c6a00e09df8cc97ff93317 -F test/walro.test 0fb4c79a9dfa1d14f46859e469d3b4844480cd9d +F test/walro.test c204f62fb6a77839bc6ce5adbdcc9df2aac877ca F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933 F test/walthread.test a25a393c068a2b42b44333fa3fdaae9072f1617c @@ -936,10 +936,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P d9bc1c7fe0ca5f6973a85827330958f4d09f8171 -R 864e9c4ee7a6cf5f8240415fb38d1af8 -T *branch * wal-readonly -T *sym-wal-readonly * -T -sym-trunk * +P bb59f9862da45d25fb51d7821130854828c91c98 +R 04ad06f1f1eb20213db6e513d8bb75aa U dan -Z 321776a9f8d852e48652ad186c3a479b +Z 112f0ce43ba9dc957c26d5146ee2fe50 diff --git a/manifest.uuid b/manifest.uuid index e39ce5418f..e0664d4e86 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb59f9862da45d25fb51d7821130854828c91c98 \ No newline at end of file +671ba5fc59f7a958e5a4138d2425b1173a442ad7 \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index 18f8823b31..4b8ea77917 100644 --- a/src/attach.c +++ b/src/attach.c @@ -76,6 +76,8 @@ static void attachFunc( Db *aNew; char *zErrDyn = 0; sqlite3_vfs *pVfs; + const char *zVfs = db->pVfs->zName; /* Name of default (main) VFS */ + int btflags = 0; UNUSED_PARAMETER(NotUsed); @@ -129,7 +131,7 @@ static void attachFunc( ** or may not be initialised. */ flags = db->openFlags; - rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); + rc = sqlite3ParseUri(zVfs, zFile, &flags, &btflags, &pVfs, &zPath, &zErr); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; sqlite3_result_error(context, zErr, -1); @@ -138,7 +140,7 @@ static void attachFunc( } assert( pVfs ); flags |= SQLITE_OPEN_MAIN_DB; - rc = sqlite3BtreeOpen(pVfs, zPath, db, &aNew->pBt, 0, flags); + rc = sqlite3BtreeOpen(pVfs, zPath, db, &aNew->pBt, btflags, flags); sqlite3_free( zPath ); db->nDb++; if( rc==SQLITE_CONSTRAINT ){ diff --git a/src/btree.h b/src/btree.h index 9e3a73b3b6..c5abc9c865 100644 --- a/src/btree.h +++ b/src/btree.h @@ -61,6 +61,7 @@ int sqlite3BtreeOpen( #define BTREE_MEMORY 4 /* This is an in-memory DB */ #define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ #define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ +#define BTREE_READONLYSHM 32 /* Read-only SHM access is acceptable */ int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); diff --git a/src/main.c b/src/main.c index 5b46f4b736..e986f83225 100644 --- a/src/main.c +++ b/src/main.c @@ -1818,6 +1818,7 @@ int sqlite3ParseUri( const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ const char *zUri, /* Nul-terminated URI to parse */ unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + int *pBtflags, /* IN/OUT: BTREE_XXX flags */ sqlite3_vfs **ppVfs, /* OUT: VFS to use */ char **pzFile, /* OUT: Filename component of URI */ char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ @@ -1933,6 +1934,12 @@ int sqlite3ParseUri( if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ zVfs = zVal; + }else if( nOpt==12 && memcmp("readonly_shm", zOpt, 12)==0 ){ + if( sqlite3Atoi(zVal) ){ + *pBtflags |= BTREE_READONLYSHM; + }else{ + *pBtflags &= ~BTREE_READONLYSHM; + } }else{ struct OpenMode { const char *z; @@ -2036,6 +2043,7 @@ static int openDatabase( int isThreadsafe; /* True for threadsafe connections */ char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ + int btflags = 0; /* Mask of BTREE_XXX flags */ *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT @@ -2163,7 +2171,8 @@ static int openDatabase( nocaseCollatingFunc, 0); /* Parse the filename/URI argument. */ - rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + rc = sqlite3ParseUri( + zVfs, zFilename, &flags, &btflags, &db->pVfs, &zOpen, &zErrMsg); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; sqlite3Error(db, rc, zErrMsg ? "%s" : 0, zErrMsg); @@ -2173,7 +2182,7 @@ static int openDatabase( /* Open the backend database driver */ db->openFlags = flags; - rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, btflags, flags | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ if( rc==SQLITE_IOERR_NOMEM ){ diff --git a/src/pager.c b/src/pager.c index 8ad6984f8a..56dae06040 100644 --- a/src/pager.c +++ b/src/pager.c @@ -620,6 +620,7 @@ struct Pager { u8 tempFile; /* zFilename is a temporary file */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ + u8 readOnlyShm; /* True if read-only shm access is Ok */ /************************************************************************** ** The following block contains those class members that change during @@ -3017,6 +3018,7 @@ static int pagerWalFrames( static int pagerBeginReadTransaction(Pager *pPager){ int rc; /* Return code */ int changed = 0; /* True if cache must be reset */ + Wal *pWal = pPager->pWal; assert( pagerUseWal(pPager) ); assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); @@ -3026,9 +3028,9 @@ static int pagerBeginReadTransaction(Pager *pPager){ ** are in locking_mode=NORMAL and EndRead() was previously called, ** the duplicate call is harmless. */ - sqlite3WalEndReadTransaction(pPager->pWal); + sqlite3WalEndReadTransaction(pWal); - rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); + rc = sqlite3WalBeginReadTransaction(pWal, pPager->readOnlyShm, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); } @@ -4414,6 +4416,7 @@ int sqlite3PagerOpen( } pPager->pVfs = pVfs; pPager->vfsFlags = vfsFlags; + pPager->readOnlyShm = (flags & PAGER_READONLYSHM)!=0; /* Open the pager file. */ diff --git a/src/pager.h b/src/pager.h index eab7ddaf80..c6cb8bcb55 100644 --- a/src/pager.h +++ b/src/pager.h @@ -60,6 +60,7 @@ typedef struct PgHdr DbPage; #define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ #define PAGER_NO_READLOCK 0x0002 /* Omit readlocks on readonly files */ #define PAGER_MEMORY 0x0004 /* In-memory database */ +#define PAGER_READONLYSHM 0x0020 /* Read-only SHM access is acceptable */ /* ** Valid values for the second argument to sqlite3PagerLockingMode(). diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4a3e45d7c0..cedd780779 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2673,7 +2673,7 @@ void sqlite3AddColumnType(Parse*,Token*); void sqlite3AddDefaultValue(Parse*,ExprSpan*); void sqlite3AddCollateType(Parse*, Token*); void sqlite3EndTable(Parse*,Token*,Token*,Select*); -int sqlite3ParseUri(const char*,const char*,unsigned int*, +int sqlite3ParseUri(const char*,const char*,unsigned int*,int*, sqlite3_vfs**,char**,char **); Bitvec *sqlite3BitvecCreate(u32); diff --git a/src/wal.c b/src/wal.c index 707fe12976..3d6baf21c8 100644 --- a/src/wal.c +++ b/src/wal.c @@ -529,7 +529,8 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); - if( rc==SQLITE_CANTOPEN && iPage==0 ){ + if( rc==SQLITE_CANTOPEN && pWal->readOnlyShm>1 ){ + assert( iPage==0 ); sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_READONLY_SHM, (void*)1); rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] @@ -541,9 +542,10 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ } } } - *ppPage = pWal->apWiData[iPage]; + assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); + if( pWal->readOnlyShm>1 ) pWal->readOnlyShm = 0; return rc; } @@ -1909,6 +1911,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ return rc; }; assert( page0 || pWal->writeLock==0 ); + assert( pWal->readOnlyShm==0 || pWal->readOnlyShm==1 ); /* If the first page of the wal-index has been mapped, try to read the ** wal-index header immediately, without holding any lock. This usually @@ -2200,13 +2203,18 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** Pager layer will use this to know that is cache is stale and ** needs to be flushed. */ -int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ +int sqlite3WalBeginReadTransaction(Wal *pWal, int readOnlyShm, int *pChanged){ int rc; /* Return code */ int cnt = 0; /* Number of TryBeginRead attempts */ + if( pWal->nWiData==0 || pWal->apWiData[0]==0 ){ + assert( readOnlyShm==0 || readOnlyShm==1 ); + pWal->readOnlyShm = readOnlyShm*2; + } do{ rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); }while( rc==WAL_RETRY ); + assert( rc || pWal->readOnlyShm==0 || (readOnlyShm && pWal->readOnlyShm==1) ); testcase( (rc&0xff)==SQLITE_BUSY ); testcase( (rc&0xff)==SQLITE_IOERR ); testcase( rc==SQLITE_PROTOCOL ); diff --git a/src/wal.h b/src/wal.h index 2039c701cf..aca5a56a56 100644 --- a/src/wal.h +++ b/src/wal.h @@ -22,7 +22,7 @@ #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 # define sqlite3WalClose(w,x,y,z) 0 -# define sqlite3WalBeginReadTransaction(y,z) 0 +# define sqlite3WalBeginReadTransaction(x,y,z) 0 # define sqlite3WalEndReadTransaction(z) # define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 @@ -56,7 +56,7 @@ int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); ** write to or checkpoint the WAL. sqlite3WalCloseSnapshot() closes the ** transaction and releases the lock. */ -int sqlite3WalBeginReadTransaction(Wal *pWal, int *); +int sqlite3WalBeginReadTransaction(Wal *pWal, int, int *); void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ diff --git a/test/walro.test b/test/walro.test index b558622cad..1025f00e33 100644 --- a/test/walro.test +++ b/test/walro.test @@ -35,6 +35,13 @@ do_multiclient_test tn { forcedelete test.db forcedelete walro + foreach c {code1 code2 code3} { + $c { + sqlite3_shutdown + sqlite3_config_uri 1 + } + } + file mkdir walro do_test 1.1.1 { @@ -49,7 +56,7 @@ do_multiclient_test tn { do_test 1.1.2 { file attributes test.db-shm -permissions r--r--r-- - code1 { sqlite3 db test.db } + code1 { sqlite3 db file:test.db?readonly_shm=1 } } {} do_test 1.1.3 { sql1 "SELECT * FROM t1" } {a b} @@ -84,7 +91,7 @@ do_multiclient_test tn { list [file exists test.db-wal] [file exists test.db-shm] } {1 1} do_test 1.2.2 { - code1 { sqlite3 db test.db } + code1 { sqlite3 db file:test.db?readonly_shm=1 } sql1 { SELECT * FROM t1 } } {a b c d e f g h i j} @@ -93,7 +100,7 @@ do_multiclient_test tn { file attributes test.db-shm -permissions rw-r--r-- hexio_write test.db-shm 0 01020304 file attributes test.db-shm -permissions r--r--r-- - code1 { sqlite3 db test.db } + code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } } {1 {attempt to write a readonly database}} do_test 1.2.4 { From a96a399ec95667f268c135e74a16bef517b1fca2 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 11 May 2011 17:36:17 +0000 Subject: [PATCH 03/69] Add missing comments associated with readonly shm changes. FossilOrigin-Name: 6a2ea52e6c09a570428161090c2f087c66f714ec --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/main.c | 5 +++++ src/sqlite.h.in | 39 ++++++++++++++++++++++++++++++++++++++- src/wal.c | 8 ++++++++ 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d15e2a3497..15dda4253a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges.\sAdd\sa\scouple\sof\sreadonly\sshm\stests. -D 2011-05-11T15:53:16.929 +C Add\smissing\scomments\sassociated\swith\sreadonly\sshm\schanges. +D 2011-05-11T17:36:17.276 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7a4d9524721d40ef9ee26f93f9bd6a51dba106f2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -144,7 +144,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 3ae0d52da013a6326310655be6473fd472347b85 -F src/main.c 951a1af49d13083da09be9d2464052e49fc25830 +F src/main.c a145cea130adfe945ab1fa7e9543c492e3f2f419 F src/malloc.c 591aedb20ae40813f1045f2ef253438a334775d9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 @@ -179,7 +179,7 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff F src/shell.c 72e7e176bf46d5c6518d15ac4ad6847c4bb5df79 -F src/sqlite.h.in 059514ec27c8f9e11c73695aa57e55d354973e1e +F src/sqlite.h.in adeb2c8019a3851a10e3872f44b34c95c6e409f2 F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 F src/sqliteInt.h 798fb09648cefc159ac9b3ce5e00f5ada1377ed1 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d @@ -245,7 +245,7 @@ F src/vdbeblob.c c3ccb7c8732858c680f442932e66ad06bb036562 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 48dcef8bc757c2e7b488f68b5ddebb1650da2450 -F src/wal.c adc6c83b650e67e32159e11d557a46594834853e +F src/wal.c 6d8ee757f62bca9b0b713724677878dff5bf7e79 F src/wal.h c1e05cdf3d42ed7c9263739ca8f5cdd8761e7de3 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 @@ -936,7 +936,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 671ba5fc59f7a958e5a4138d2425b1173a442ad7 3e490915301216e242a5cdeb0febaff12ed53cb9 -R 1feba9025b47c541dc5cefaffc74e97e +P cde45a033ee6834900f5f5c272c383408883a74c +R 77f3671cb0a1ec2771484489a2a100be U dan -Z af31082504210cb42f7a67f58917a8ef +Z 826df29e354d14eb64673fd89c39c62e diff --git a/manifest.uuid b/manifest.uuid index 8bb70d0669..68e3f5666d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cde45a033ee6834900f5f5c272c383408883a74c \ No newline at end of file +6a2ea52e6c09a570428161090c2f087c66f714ec \ No newline at end of file diff --git a/src/main.c b/src/main.c index 45ee8fbd5b..f4cf735e3c 100644 --- a/src/main.c +++ b/src/main.c @@ -1803,6 +1803,11 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ ** *pFlags may be updated before returning if the URI filename contains ** "cache=xxx" or "mode=xxx" query parameters. ** +** The third argument, pBtflags, points to an integer containing the flags +** that will be passed as the 5th argument to sqlite3BtreeOpen (BTREE_XXX +** flags). This value will be edited if the URI filename contains a +** "readonly_shm=1" or "readonly_shm=0" query parameter. +** ** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to ** the VFS that should be used to open the database file. *pzFile is set to ** point to a buffer containing the name of the file to open. It is the diff --git a/src/sqlite.h.in b/src/sqlite.h.in index bafd322264..a002a8b4ea 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -736,6 +736,27 @@ struct sqlite3_io_methods { ** Applications should not call [sqlite3_file_control()] with this ** opcode as doing so may disrupt the operation of the specialized VFSes ** that do require it. +** +** The [SQLITE_FCNTL_READONLY_SHM] may be generated internally by SQLite if +** the "readonly_shm=1" URI option is specified when the database is opened. +** The fourth argument passed to the VFS xFileControl methods is a pointer +** to a variable of type "int" containing the value 1 or 0. If the variable +** contains the value 1, then this indicates to the VFS that a read-only +** mapping of the shared-memory region is acceptable. If it is set to 0, then +** this indicates that a read-write mapping is required (as normal). If +** a read-only mapping is returned, then the VFS may also return read-only +** mappings for any subsequent requests via the same file-descriptor - +** regardless of the value most recently configured using +** SQLITE_FCNTL_READONLY_SHM. +** +** In practice, if "readonly_shm=1" is specified and the first attempt to +** map a shared-memory region fails, then this file-control is invoked with +** the argument variable set to 1 and a second attempt to map the shared-memory +** region is made. If this mapping succeeds, then the connection continues +** with the read-only mapping. Otherwise, if it fails, SQLITE_CANTOPEN is +** returned to the caller. Whether or not the second (read-only) mapping +** attempt succeeds, the file-control is invoked again with the argument +** variable set to 0. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 @@ -2444,7 +2465,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ** The query component of a URI may contain parameters that are interpreted ** either by SQLite itself, or by a [sqlite3_vfs | custom VFS implementation]. -** SQLite interprets the following three query parameters: +** SQLite interprets the following four query parameters: ** **
    **
  • vfs: ^The "vfs" parameter may be used to specify the name of @@ -2476,6 +2497,22 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in ** a URI filename, its value overrides any behaviour requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
  • readonly_shm: ^The readonly_shm parameter may be set to +** either "1" or "0". Setting it to "1" indicates that if the database +** is in WAL mode and read-write access to the associated shared +** memory region cannot be obtained, then an attempt should be made to open +** the shared-memory in read-only mode instead. If there exist one or +** more other database clients that have read-write connections to the +** database shared-memory, then a read-only shared-memory connection works +** fine. However, if there exist no clients with read-write connections +** to the shared-memory and the most recent such crashed or was interrupted +** by a power failure, then it is possible for a database client using a +** read-only connection to return incorrect data, incorrectly report +** database corruption, or return an SQLITE_READONLY error. Or if the +** most recent read-write connection shut down cleanly, it may not be +** possible to open the shared-memory in read-only mode at all, and SQLite +** will return SQLITE_CANTOPEN. **
** ** ^Specifying an unknown parameter in the query component of a URI is not an diff --git a/src/wal.c b/src/wal.c index 3d6baf21c8..2ee7f59e89 100644 --- a/src/wal.c +++ b/src/wal.c @@ -406,6 +406,14 @@ struct WalCkptInfo { /* ** An open write-ahead log file is represented by an instance of the ** following object. +** +** The readOnlyShm variable is normally set to 0. If it is set to 1, then +** the connection to shared-memory is read-only. This means it cannot +** be written at all (even when read-locking the database). If it is set +** to 2, then the shared-memory region is not yet open, but a read-only +** connection is acceptable. In this case when the shared-memory is opened +** (see function walIndexPage()), readOnlyShm is set to either 0 or 1 as +** appropriate. */ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ From 09643ab745aee64145ddacf3579f9c5eaf6a4452 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 24 May 2011 18:49:45 +0000 Subject: [PATCH 04/69] If the fts4 option prefix=1 is specified, have the fts4 module maintain an index of prefixes as well as terms. FossilOrigin-Name: b5bdc639898ee22eebedeb560810e94e74de8aa4 --- ext/fts3/fts3.c | 105 +++++++++++++------- ext/fts3/fts3Int.h | 30 +++++- ext/fts3/fts3_aux.c | 2 +- ext/fts3/fts3_term.c | 15 ++- ext/fts3/fts3_write.c | 222 ++++++++++++++++++++++++++++++++---------- manifest | 26 ++--- manifest.uuid | 2 +- test/fts3prefix.test | 175 +++++++++++++++++++++++++++++++++ 8 files changed, 469 insertions(+), 108 deletions(-) create mode 100644 test/fts3prefix.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index fecf092148..a127773449 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -865,6 +865,7 @@ static int fts3InitVtab( int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ int bNoDocsize = 0; /* True to omit %_docsize table */ + int bPrefix = 0; /* True to include a prefix-search index */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ @@ -927,6 +928,8 @@ static int fts3InitVtab( }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ zUncompress = zVal; zVal = 0; + }else if( nKey==6 && 0==sqlite3_strnicmp(z, "prefix", 6) ){ + bPrefix = 1; }else{ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); rc = SQLITE_ERROR; @@ -979,7 +982,9 @@ static int fts3InitVtab( p->bHasStat = isFts4; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); + p->bPrefix = bPrefix; fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); + fts3HashInit(&p->pendingPrefixes, FTS3_HASH_STRING, 1); /* Fill in the zName and zDb fields of the vtab structure. */ zCsr = (char *)&p->azColumn[nCol]; @@ -2139,6 +2144,28 @@ static int fts3DeferredTermSelect( return SQLITE_OK; } +static int fts3SegReaderCursorAppend( + Fts3SegReaderCursor *pCsr, + Fts3SegReader *pNew +){ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; + } + pCsr->apSegment[pCsr->nSegment++] = pNew; + return SQLITE_OK; +} + +/* +** Set up a cursor object for iterating through the full-text index or +** a single level therein. +*/ int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iLevel, /* Level of segments to scan */ @@ -2152,42 +2179,52 @@ int sqlite3Fts3SegReaderCursor( int rc2; int iAge = 0; sqlite3_stmt *pStmt = 0; - Fts3SegReader *pPending = 0; - assert( iLevel==FTS3_SEGCURSOR_ALL + assert( iLevel==FTS3_SEGCURSOR_ALL_TERM || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX + || iLevel==FTS3_SEGCURSOR_ALL_PREFIX || iLevel>=0 ); - assert( FTS3_SEGCURSOR_PENDING<0 ); - assert( FTS3_SEGCURSOR_ALL<0 ); - assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); + assert( 0>FTS3_SEGCURSOR_ALL_TERM + && 0>FTS3_SEGCURSOR_PENDING + && 0>FTS3_SEGCURSOR_PENDING_PREFIX + && 0>FTS3_SEGCURSOR_ALL_PREFIX + ); + assert( iLevel==FTS3_SEGCURSOR_ALL_TERM + || iLevel==FTS3_SEGCURSOR_ALL_PREFIX + || (zTerm==0 && isPrefix==1) + ); assert( isPrefix==0 || isScan==0 ); - memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); - /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ + /* "isScan" is only set to true by the ft4aux module, not an ordinary + ** full-text table. The pendingTerms and pendingPrefixes tables must be + ** empty in this case. */ assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 ); + assert( isScan==0 || fts3HashCount(&p->pendingPrefixes)==0 ); + + /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ if( iLevel<0 && isScan==0 ){ - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending); + int bPrefix = ( + iLevel==FTS3_SEGCURSOR_PENDING_PREFIX + || iLevel==FTS3_SEGCURSOR_ALL_PREFIX + ); + Fts3SegReader *pPending = 0; + + rc = sqlite3Fts3SegReaderPending(p,zTerm,nTerm,isPrefix,bPrefix,&pPending); if( rc==SQLITE_OK && pPending ){ - int nByte = (sizeof(Fts3SegReader *) * 16); - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); - if( pCsr->apSegment==0 ){ - rc = SQLITE_NOMEM; - }else{ - pCsr->apSegment[0] = pPending; - pCsr->nSegment = 1; - pPending = 0; - } + rc = fts3SegReaderCursorAppend(pCsr, pPending); } } - if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( iLevel!=FTS3_SEGCURSOR_PENDING && iLevel!=FTS3_SEGCURSOR_PENDING_PREFIX ){ if( rc==SQLITE_OK ){ rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); } while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; /* Read the values returned by the SELECT into local variables. */ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); @@ -2196,18 +2233,6 @@ int sqlite3Fts3SegReaderCursor( int nRoot = sqlite3_column_bytes(pStmt, 4); char const *zRoot = sqlite3_column_blob(pStmt, 4); - /* If nSegment is a multiple of 16 the array needs to be extended. */ - if( (pCsr->nSegment%16)==0 ){ - Fts3SegReader **apNew; - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); - if( !apNew ){ - rc = SQLITE_NOMEM; - goto finished; - } - pCsr->apSegment = apNew; - } - /* If zTerm is not NULL, and this segment is not stored entirely on its ** root node, the range of leaves scanned can be reduced. Do this. */ if( iStartBlock && zTerm ){ @@ -2218,10 +2243,10 @@ int sqlite3Fts3SegReaderCursor( } rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment] + iEndBlock, zRoot, nRoot, &pSeg ); if( rc!=SQLITE_OK ) goto finished; - pCsr->nSegment++; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); iAge++; } } @@ -2229,7 +2254,6 @@ int sqlite3Fts3SegReaderCursor( finished: rc2 = sqlite3_reset(pStmt); if( rc==SQLITE_DONE ) rc = rc2; - sqlite3Fts3SegReaderFree(pPending); return rc; } @@ -2247,11 +2271,18 @@ static int fts3TermSegReaderCursor( pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); if( pSegcsr ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int i; int nCost = 0; - rc = sqlite3Fts3SegReaderCursor( - p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr); + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + + if( isPrefix && p->bPrefix && nTerm<=FTS3_MAX_PREFIX ){ + rc = sqlite3Fts3SegReaderCursor( + p, FTS3_SEGCURSOR_ALL_PREFIX, zTerm, nTerm, 0, 0, pSegcsr); + + }else{ + rc = sqlite3Fts3SegReaderCursor( + p, FTS3_SEGCURSOR_ALL_TERM, zTerm, nTerm, isPrefix, 0, pSegcsr); + } for(i=0; rc==SQLITE_OK && inSegment; i++){ rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); @@ -3309,6 +3340,7 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ assert( p->inTransaction!=0 ); TESTONLY( p->inTransaction = 0 ); TESTONLY( p->mxSavepoint = -1; ); + sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab); return SQLITE_OK; } @@ -3683,6 +3715,7 @@ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); sqlite3Fts3PendingTermsClear(p); + sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab); return SQLITE_OK; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index bb21165ffc..785a6aa655 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -23,6 +23,8 @@ #include "fts3_tokenizer.h" #include "fts3_hash.h" +#define FTS3_MAX_PREFIX 8 + /* ** This constant controls how often segments are merged. Once there are ** FTS3_MERGE_COUNT segments of level N, they are merged into a single @@ -53,6 +55,20 @@ */ #define FTS3_VARINT_MAX 10 +/* +** FTS4 virtual tables may maintain two separate indexes. One that indexes +** all document terms (the same index FTS3 tables maintain) and another used +** for prefixes. B+-trees that are part of the prefix index have values for +** the %_segdir.level column that are equal to or greater than the following +** value. +** +** It is considered impossible for the regular index to use levels this large. +** In theory it could, but that would require that at least 2^1024 separate +** write operations to be made within the lifetime of the database. +*/ +#define FTS3_SEGDIR_PREFIXLEVEL 1024 +#define FTS3_SEGDIR_PREFIXLEVEL_STR "1024" + /* ** The testcase() macro is only used by the amalgamation. If undefined, ** make it a no-op. @@ -148,7 +164,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[24]; + sqlite3_stmt *aStmt[27]; char *zReadExprlist; char *zWriteExprlist; @@ -156,6 +172,7 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bPrefix; /* True if there is a prefix index */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ @@ -171,6 +188,7 @@ struct Fts3Table { int nPendingData; sqlite_int64 iPrevDocid; Fts3Hash pendingTerms; + Fts3Hash pendingPrefixes; #if defined(SQLITE_DEBUG) /* State variables used for validating that the transaction control @@ -317,7 +335,8 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); -int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); +int sqlite3Fts3SegReaderPending( + Fts3Table*,const char*,int,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); @@ -334,8 +353,11 @@ void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); void sqlite3Fts3SegmentsClose(Fts3Table *); -#define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_ALL -2 +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_PENDING_PREFIX -2 +#define FTS3_SEGCURSOR_ALL_PREFIX -3 +#define FTS3_SEGCURSOR_ALL_TERM -4 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 6108689ae1..cb91cda347 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -375,7 +375,7 @@ static int fts3auxFilterMethod( if( pCsr->zStop==0 ) return SQLITE_NOMEM; } - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL_TERM, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index e207ff870d..ca46269689 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -27,6 +27,7 @@ typedef struct Fts3termCursor Fts3termCursor; struct Fts3termTable { sqlite3_vtab base; /* Base class used by SQLite core */ + int bPrefix; /* True for an fts4prefix table */ Fts3Table *pFts3Tab; }; @@ -56,7 +57,7 @@ struct Fts3termCursor { */ static int fts3termConnectMethod( sqlite3 *db, /* Database connection */ - void *pUnused, /* Unused */ + void *pCtx, /* Non-zero for an fts4prefix table */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ @@ -70,8 +71,6 @@ static int fts3termConnectMethod( int rc; /* value returned by declare_vtab() */ Fts3termTable *p; /* Virtual table object to return */ - UNUSED_PARAMETER(pUnused); - /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ *pzErr = sqlite3_mprintf( @@ -97,6 +96,7 @@ static int fts3termConnectMethod( p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; + p->bPrefix = (int)pCtx; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); @@ -244,7 +244,8 @@ static int fts3termFilterMethod( sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; - Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab; + Fts3termTable *p = (Fts3termTable *)pCursor->pVtab; + Fts3Table *pFts3 = p->pFts3Tab; int rc; UNUSED_PARAMETER(nVal); @@ -262,7 +263,8 @@ static int fts3termFilterMethod( pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, + p->bPrefix ? FTS3_SEGCURSOR_ALL_PREFIX : FTS3_SEGCURSOR_ALL_TERM, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr ); if( rc==SQLITE_OK ){ @@ -353,6 +355,9 @@ int sqlite3Fts3InitTerm(sqlite3 *db){ int rc; /* Return code */ rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "fts4prefix", &fts3term_module, (void*)1); + } return rc; } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index d00632a75d..5ff7e0b1aa 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -145,6 +145,14 @@ struct SegmentWriter { ** fts3NodeAddTerm() ** fts3NodeWrite() ** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. */ struct SegmentNode { SegmentNode *pParent; /* Parent node (or NULL for root node) */ @@ -177,7 +185,7 @@ struct SegmentNode { #define SQL_SELECT_LEVEL 12 #define SQL_SELECT_ALL_LEVEL 13 #define SQL_SELECT_LEVEL_COUNT 14 -#define SQL_SELECT_SEGDIR_COUNT_MAX 15 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 #define SQL_DELETE_SEGDIR_BY_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 @@ -186,6 +194,10 @@ struct SegmentNode { #define SQL_SELECT_DOCSIZE 21 #define SQL_SELECT_DOCTOTAL 22 #define SQL_REPLACE_DOCTOTAL 23 +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 + +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 +#define SQL_DELETE_ALL_PREFIX_SEGDIR 26 /* ** This function is used to obtain an SQLite prepared statement handle @@ -222,10 +234,12 @@ static int fts3SqlStmt( /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", + "FROM %Q.'%q_segdir' WHERE level < " FTS3_SEGDIR_PREFIXLEVEL_STR + " ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1)*" + FTS3_SEGDIR_PREFIXLEVEL_STR, /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", @@ -235,6 +249,11 @@ static int fts3SqlStmt( /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 24 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level >= " FTS3_SEGDIR_PREFIXLEVEL_STR + " ORDER BY level DESC, idx ASC", +/* 25 */ "DELETE FROM %Q.'%q_segdir' WHERE level<" FTS3_SEGDIR_PREFIXLEVEL_STR, +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level>=" FTS3_SEGDIR_PREFIXLEVEL_STR, }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -393,9 +412,12 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ int rc; sqlite3_stmt *pStmt = 0; - if( iLevel<0 ){ + if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_PREFIX_LEVEL, &pStmt, 0); + }else if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); }else{ + assert( iLevel>=0 ); rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); } @@ -512,6 +534,40 @@ static int fts3PendingListAppend( return 0; } +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + + + + /* ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in @@ -560,8 +616,6 @@ static int fts3PendingTermsAdd( while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ - PendingList *pList; - if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. @@ -572,23 +626,13 @@ static int fts3PendingTermsAdd( break; } - pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); - if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); - } - if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ - if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ - /* Malloc failed while inserting the new entry. This can only - ** happen if there was no previous entry for this token. - */ - assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); - sqlite3_free(pList); - rc = SQLITE_NOMEM; + rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingTerms,zToken,nToken); + if( p->bPrefix ){ + int n = (nToken > FTS3_MAX_PREFIX ? FTS3_MAX_PREFIX : nToken); + for(; n>0 && rc==SQLITE_OK; n--){ + rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingPrefixes,zToken,n); } } - if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); - } } pModule->xClose(pCsr); @@ -628,6 +672,17 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ p->nPendingData = 0; } +/* +** Discard the contents of the pending-prefixes hash table. +*/ +void sqlite3Fts3PendingPrefixesClear(Fts3Table *p){ + Fts3HashElem *pElem; + for(pElem=fts3HashFirst(&p->pendingPrefixes); pElem; pElem=fts3HashNext(pElem)){ + sqlite3_free(fts3HashData(pElem)); + } + fts3HashClear(&p->pendingPrefixes); +} + /* ** This function is called by the xUpdate() method as part of an INSERT ** operation. It adds entries for each term in the new record to the @@ -725,6 +780,7 @@ static int fts3DeleteAll(Fts3Table *p){ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); + sqlite3Fts3PendingPrefixesClear(p); /* Delete everything from the %_content, %_segments and %_segdir tables. */ fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); @@ -1235,12 +1291,28 @@ static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ /* ** This function is used to allocate an Fts3SegReader that iterates through ** a subset of the terms stored in the Fts3Table.pendingTerms array. +** +** If the isPrefixIter parameter is zero, then the returned SegReader iterates +** through each term in the pending-terms table. Or, if isPrefixIter is +** non-zero, it iterates through each term and its prefixes. For example, if +** the pending terms hash table contains the terms "sqlite", "mysql" and +** "firebird", then the iterator visits the following 'terms' (in the order +** shown): +** +** f fi fir fire fireb firebi firebir firebird +** m my mys mysq mysql +** s sq sql sqli sqlit sqlite +** +** Whereas if isPrefixIter is zero, the terms visited are: +** +** firebird mysql sqlite */ int sqlite3Fts3SegReaderPending( Fts3Table *p, /* Virtual table handle */ const char *zTerm, /* Term to search for */ int nTerm, /* Size of buffer zTerm */ - int isPrefix, /* True for a term-prefix query */ + int isMultiTerm, /* True to visit multiple terms */ + int isPrefixIter, /* 0->pendingTerms, 1->pendingPrefixes */ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ @@ -1248,11 +1320,14 @@ int sqlite3Fts3SegReaderPending( int nElem = 0; /* Size of array at aElem */ int rc = SQLITE_OK; /* Return Code */ - if( isPrefix ){ + if( isMultiTerm ){ int nAlloc = 0; /* Size of allocated array at aElem */ Fts3HashElem *pE = 0; /* Iterator variable */ + Fts3Hash *pHash; - for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){ + pHash = (isPrefixIter ? &p->pendingPrefixes : &p->pendingTerms); + + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); int nKey = fts3HashKeysize(pE); if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ @@ -1269,6 +1344,7 @@ int sqlite3Fts3SegReaderPending( } aElem = aElem2; } + aElem[nElem++] = pE; } } @@ -1282,6 +1358,8 @@ int sqlite3Fts3SegReaderPending( } }else{ + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. */ Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); if( pE ){ aElem = &pE; @@ -1302,7 +1380,7 @@ int sqlite3Fts3SegReaderPending( } } - if( isPrefix ){ + if( isMultiTerm ){ sqlite3_free(aElem); } *ppReader = pReader; @@ -1914,21 +1992,30 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ } /* -** Set *pnSegment to the total number of segments in the database. Set -** *pnMax to the largest segment level in the database (segment levels -** are stored in the 'level' column of the %_segdir table). +** Set *pnMax to the largest segment level in the database for either the +** terms index (if parameter bPrefixIndex is 0) or the prefixes index (if +** parameter bPrefixIndex is 1). +** +** Segment levels are stored in the 'level' column of the %_segdir table. ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ +static int fts3SegmentMaxLevel(Fts3Table *p, int bPrefixIndex, int *pnMax){ sqlite3_stmt *pStmt; int rc; + assert( bPrefixIndex==0 || bPrefixIndex==1 ); - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1) * 1024 + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int(pStmt, 1, bPrefixIndex); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnSegment = sqlite3_column_int(pStmt, 0); - *pnMax = sqlite3_column_int(pStmt, 1); + *pnMax = sqlite3_column_int(pStmt, 0); } return sqlite3_reset(pStmt); } @@ -1971,12 +2058,21 @@ static int fts3DeleteSegdir( return rc; } - if( iLevel==FTS3_SEGCURSOR_ALL ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + assert( iLevel>=0 + || iLevel==FTS3_SEGCURSOR_ALL_TERM + || iLevel==FTS3_SEGCURSOR_ALL_PREFIX + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX + ); + if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_TERMS_SEGDIR, 0); + }else if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_PREFIX_SEGDIR, 0); + }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){ + sqlite3Fts3PendingPrefixesClear(p); }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ sqlite3Fts3PendingTermsClear(p); - }else{ - assert( iLevel>=0 ); + }else if( iLevel>=0 ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pDelete, 1, iLevel); @@ -2234,27 +2330,34 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; - if( iLevel==FTS3_SEGCURSOR_ALL ){ + if( iLevel==FTS3_SEGCURSOR_ALL_TERM || iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically ** greatest segment level currently present in the database. The index ** of the new segment is always 0. */ - int nDummy; /* TODO: Remove this */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel); + rc = fts3SegmentMaxLevel(p, iLevel==FTS3_SEGCURSOR_ALL_PREFIX, &iNewLevel); + bIgnoreEmpty = 1; }else{ - /* This call is to merge all segments at level iLevel. Find the next + /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - iNewLevel = iLevel+1; + if( iLevel==FTS3_SEGCURSOR_PENDING ){ + iNewLevel = 0; + }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){ + iNewLevel = FTS3_SEGDIR_PREFIXLEVEL; + }else{ + iNewLevel = iLevel+1; + } rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); } if( rc!=SQLITE_OK ) goto finished; @@ -2263,7 +2366,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; - filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0); + filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); while( SQLITE_OK==rc ){ @@ -2290,7 +2393,17 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ ** Flush the contents of pendingTerms to a level 0 segment. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ - return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); + int rc = SQLITE_OK; + if( p->bPrefix ){ + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING_PREFIX); + } + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + return rc; } /* @@ -2457,11 +2570,9 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - }else{ - sqlite3Fts3PendingTermsClear(p); + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX); + if( rc==SQLITE_OK ){ + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM); } #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ @@ -2476,6 +2587,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ } sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + sqlite3Fts3PendingPrefixesClear(p); return rc; } @@ -2792,13 +2905,22 @@ int sqlite3Fts3UpdateMethod( */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; + int bReturnDone = 0; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX); + if( rc==SQLITE_OK ){ + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM); + } + if( rc==SQLITE_DONE ){ + bReturnDone = 1; + rc = SQLITE_OK; + } if( rc==SQLITE_OK ){ rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); if( rc==SQLITE_OK ){ sqlite3Fts3PendingTermsClear(p); + sqlite3Fts3PendingPrefixesClear(p); } }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); @@ -2806,7 +2928,7 @@ int sqlite3Fts3Optimize(Fts3Table *p){ } } sqlite3Fts3SegmentsClose(p); - return rc; + return ((rc==SQLITE_OK && bReturnDone) ? SQLITE_DONE : rc); } #endif diff --git a/manifest b/manifest index 51c28b73df..bf93a84328 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sinvoke\sthe\sxRollbackTo\sor\sxRelease\smethods\sof\sa\svirtual\stable\swithout\nhaving\sfirst\sinvoked\san\sappropriate\sxSavepoint\smethod.\s\sAdd\sassert()\sstatements\nto\sFTS3/4\sto\sverify\sthat\sthis\sis\shappening\sin\sall\scases. -D 2011-05-24T15:36:01.532 +C If\sthe\sfts4\soption\sprefix=1\sis\sspecified,\shave\sthe\sfts4\smodule\smaintain\san\sindex\sof\sprefixes\sas\swell\sas\sterms. +D 2011-05-24T18:49:45.786 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c e7600e285b58027657ebb46c6132b7d5a100accb +F ext/fts3/fts3.c 4a48bfef342badba0a71bdeb5354edaa3ad83382 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h c8c0011c5e5b3a7703376ea6cd7deb91cfb96a06 -F ext/fts3/fts3_aux.c 9e931f55eed8498dafe7bc1160f10cbb1a652fdf +F ext/fts3/fts3Int.h 02699211c0b6cf5aa713cc68c527c9a6e9159fbe +F ext/fts3/fts3_aux.c d68d8e4d39e0342302d2c834618755af7c8058ea F ext/fts3/fts3_expr.c 5f49e0deaf723724b08100bb3ff40aab02ad0c93 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 F ext/fts3/fts3_snippet.c 92b40397b28422c35c4127492d7ac6da34d1966a -F ext/fts3/fts3_term.c f115f5a5f4298303d3b22fc6c524b8d565c7b950 +F ext/fts3/fts3_term.c cd226a311940b8ef414d5c1f7c74971a47cacedb F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c b50181e5ecf484c2f56e98d651424e4b69f96c89 +F ext/fts3/fts3_write.c 0fd6a55c774731852f889007fc6edb1b99819ee5 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -473,6 +473,7 @@ F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 +F test/fts3prefix.test 5b4e08c63d5d4a79e54754dc6b2209b03c885200 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 @@ -938,7 +939,10 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a9d095660ca0e99b226e0fe669c11a0be6c49710 -R 19875cc50e6351fd8eaf432c5942eda7 -U drh -Z f15ff60c160c7edadf458f2c05939bd4 +P 651ef24249d8c22c4f13e4c0bb98a60099cfd23a +R c499b030bb5d74c4afca730dec7a05bc +T *branch * fts3-prefix-search +T *sym-fts3-prefix-search * +T -sym-trunk * +U dan +Z 0c60a0f2885eb6df2e24e22407faffbe diff --git a/manifest.uuid b/manifest.uuid index 47cc55cc06..747bc2e8b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -651ef24249d8c22c4f13e4c0bb98a60099cfd23a \ No newline at end of file +b5bdc639898ee22eebedeb560810e94e74de8aa4 \ No newline at end of file diff --git a/test/fts3prefix.test b/test/fts3prefix.test new file mode 100644 index 0000000000..95a5c89629 --- /dev/null +++ b/test/fts3prefix.test @@ -0,0 +1,175 @@ +# 2011 May 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts3prefix + +# This proc tests that the prefixes index appears to represent the same content +# as the terms index. +# +proc fts3_terms_and_prefixes {db tbl} { + $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl);" + $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4prefix($tbl);" + + $db eval { + CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; + CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; + CREATE INDEX temp.idx ON prefixes(term); + DROP TABLE fts3check1; + DROP TABLE fts3check2; + } + + $db eval { SELECT term, docid, col, pos FROM temp.terms } a { + set nMax [expr [string length $a(term)] - 1] + if {$nMax>8} {set nMax 8} + for {set n 0} {$n < $nMax} {incr n} { + set t [string range $a(term) 0 $n] + set r [$db one { + SELECT count(*) FROM temp.prefixes WHERE + term = $t AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) + }] + if {$r != 1} { + error "$t, $a(docid), $a(col), $a(pos)" + } + } + } + + execsql { DROP TABLE temp.prefixes } + execsql { DROP TABLE temp.terms } + + set terms_layout [$db eval " + SELECT level, idx FROM ${tbl}_segdir WHERE level < 1024 ORDER by 1, 2 + "] + set prefixes_layout [$db eval " + SELECT level-1024, idx FROM ${tbl}_segdir WHERE level >= 1024 ORDER by 1, 2 + "] + + if {$terms_layout != $prefixes_layout} { + puts "TERMS LAYOUT: $terms_layout" + puts "PREFIX LAYOUT: $prefixes_layout" + error "Terms and prefixes are comprised of different b-trees" + } + + return "" +} +proc fts3_tap_test {tn db tbl} { + uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl] ""] +} + +#------------------------------------------------------------------------- +# Test cases 1.* are a sanity check. They test that the prefixes index is +# being constructed correctly for the simplest possible case. +# +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts4(prefix=1); + CREATE VIRTUAL TABLE prefixes USING fts4prefix(t1); + CREATE VIRTUAL TABLE terms USING fts4term(t1); +} +do_execsql_test 1.2 { + INSERT INTO t1 VALUES('sqlite mysql firebird'); +} +do_execsql_test 1.3 { + SELECT term FROM prefixes; +} {f fi fir fire fireb firebi firebir firebird m my mys mysq mysql s sq sql sqli sqlit sqlite} +do_execsql_test 1.4 { + SELECT term FROM terms; +} {firebird mysql sqlite} + +fts3_tap_test 1.5 db t1 + +#------------------------------------------------------------------------- +# A slightly more complicated dataset. This test also verifies that DELETE +# operations do not corrupt the prefixes index. +# +do_execsql_test 2.1 { + INSERT INTO t1 VALUES('FTS3 and FTS4 are an SQLite virtual table modules'); + INSERT INTO t1 VALUES('that allows users to perform full-text searches on'); + INSERT INTO t1 VALUES('a set of documents. The most common (and'); + INSERT INTO t1 VALUES('effective) way to describe full-text searches is'); + INSERT INTO t1 VALUES('"what Google, Yahoo and Altavista do with'); + INSERT INTO t1 VALUES('documents placed on the World Wide Web". Users'); + INSERT INTO t1 VALUES('input a term, or series of terms, perhaps'); + INSERT INTO t1 VALUES('connected by a binary operator or grouped together'); + INSERT INTO t1 VALUES('into a phrase, and the full-text query system'); + INSERT INTO t1 VALUES('finds the set of documents that best matches those'); + INSERT INTO t1 VALUES('terms considering the operators and groupings the'); + INSERT INTO t1 VALUES('user has specified. This article describes the'); + INSERT INTO t1 VALUES('deployment and usage of FTS3 and FTS4.'); + INSERT INTO t1 VALUES('FTS1 and FTS2 are obsolete full-text search'); + INSERT INTO t1 VALUES('modules for SQLite. There are known issues with'); + INSERT INTO t1 VALUES('these older modules and their use should be'); + INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were'); + INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of'); + INSERT INTO t1 VALUES('Google. It is now developed and maintained as part'); + INSERT INTO t1 VALUES('of SQLite. '); +} +fts3_tap_test 2.2 db t1 +do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; } +fts3_tap_test 2.4 db t1 + +do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') } +fts3_tap_test 2.6 db t1 + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t2 USING fts4(prefix=1); + INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); + INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); + INSERT INTO t2 VALUES('fleet was forced to take shelter at'); + INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind'); + INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet'); + INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey'); + INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); + INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); + INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); + INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); + INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); + INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); + INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); + INSERT INTO t2 VALUES('fleet was forced to take shelter at'); + INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind'); + INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet'); + INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey'); + INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); + INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); + INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); + INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); + INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); +} + +fts3_tap_test 3.2 db t2 +do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}} +fts3_tap_test 3.4 db t2 + + +#------------------------------------------------------------------------- +# Simple tests for reading the prefix-index. +# +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE t3 USING fts4(prefix=1); + INSERT INTO t3 VALUES('one two three'); + INSERT INTO t3 VALUES('four five six'); + INSERT INTO t3 VALUES('seven eight nine'); +} +do_execsql_test 4.2 { + SELECT * FROM t3 WHERE t3 MATCH 'f*' +} {{four five six}} +do_execsql_test 4.3 { + SELECT * FROM t3 WHERE t3 MATCH 'four*' +} {{four five six}} +do_execsql_test 4.4 { + SELECT * FROM t3 WHERE t3 MATCH 's*' +} {{four five six} {seven eight nine}} + +finish_test From 9d1f874ab30b038d7e0bda416dfeee6ecf1be790 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 25 May 2011 18:34:53 +0000 Subject: [PATCH 05/69] Change fts4 so that the prefix= parameter is passes a comma-separated list of integers. For each integer N, a separate index of all prefixes of length N bytes is created. FossilOrigin-Name: be59bf49402d2e2f4b95fb6668849f3745cb7bf2 --- ext/fts3/fts3.c | 174 +++++++++++++++++-------- ext/fts3/fts3Int.h | 59 +++++---- ext/fts3/fts3_aux.c | 3 +- ext/fts3/fts3_term.c | 17 +-- ext/fts3/fts3_write.c | 286 ++++++++++++++++++++++-------------------- manifest | 25 ++-- manifest.uuid | 2 +- test/fts3prefix.test | 110 +++++++++------- 8 files changed, 400 insertions(+), 276 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index a127773449..6a5c2ae4ef 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -832,6 +832,58 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ return zRet; } +static int fts3GobbleInt(const char **pp, int *pnOut){ + const char *p = *pp; + int nInt = 0; + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ + nInt = nInt * 10 + (p[0] - '0'); + } + if( p==*pp ) return SQLITE_ERROR; + *pnOut = nInt; + *pp = p; + return SQLITE_OK; +} + + +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ + struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ +){ + struct Fts3Index *aIndex; + int nIndex = 1; + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + *apIndex = *apFree = aIndex; + *pnIndex = nIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; ibHasStat = isFts4; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); - p->bPrefix = bPrefix; - fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); - fts3HashInit(&p->pendingPrefixes, FTS3_HASH_STRING, 1); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->azColumn[nCol]; + zCsr = (char *)&p->aIndex[nIndex]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; @@ -1034,6 +1104,8 @@ static int fts3InitVtab( fts3DeclareVtab(&rc, p); fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); @@ -2144,6 +2216,9 @@ static int fts3DeferredTermSelect( return SQLITE_OK; } +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ static int fts3SegReaderCursorAppend( Fts3SegReaderCursor *pCsr, Fts3SegReader *pNew @@ -2163,11 +2238,12 @@ static int fts3SegReaderCursorAppend( } /* -** Set up a cursor object for iterating through the full-text index or -** a single level therein. +** Set up a cursor object for iterating through a full-text index or a +** single level therein. */ int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ @@ -2180,49 +2256,41 @@ int sqlite3Fts3SegReaderCursor( int iAge = 0; sqlite3_stmt *pStmt = 0; - assert( iLevel==FTS3_SEGCURSOR_ALL_TERM + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX || iLevel>=0 ); - assert( 0>FTS3_SEGCURSOR_ALL_TERM - && 0>FTS3_SEGCURSOR_PENDING - && 0>FTS3_SEGCURSOR_PENDING_PREFIX - && 0>FTS3_SEGCURSOR_ALL_PREFIX - ); - assert( iLevel==FTS3_SEGCURSOR_ALL_TERM - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX - || (zTerm==0 && isPrefix==1) - ); + assert( iLevelaIndex==0 ); + memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); - /* "isScan" is only set to true by the ft4aux module, not an ordinary - ** full-text table. The pendingTerms and pendingPrefixes tables must be - ** empty in this case. */ - assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 ); - assert( isScan==0 || fts3HashCount(&p->pendingPrefixes)==0 ); - - /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ - if( iLevel<0 && isScan==0 ){ - int bPrefix = ( - iLevel==FTS3_SEGCURSOR_PENDING_PREFIX - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX - ); - Fts3SegReader *pPending = 0; - - rc = sqlite3Fts3SegReaderPending(p,zTerm,nTerm,isPrefix,bPrefix,&pPending); - if( rc==SQLITE_OK && pPending ){ - rc = fts3SegReaderCursorAppend(pCsr, pPending); + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); } } - if( iLevel!=FTS3_SEGCURSOR_PENDING && iLevel!=FTS3_SEGCURSOR_PENDING_PREFIX ){ + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); + rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); } + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ Fts3SegReader *pSeg = 0; @@ -2273,17 +2341,25 @@ static int fts3TermSegReaderCursor( if( pSegcsr ){ int i; int nCost = 0; + int bFound = 0; /* True once an index has been found */ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - if( isPrefix && p->bPrefix && nTerm<=FTS3_MAX_PREFIX ){ - rc = sqlite3Fts3SegReaderCursor( - p, FTS3_SEGCURSOR_ALL_PREFIX, zTerm, nTerm, 0, 0, pSegcsr); - - }else{ - rc = sqlite3Fts3SegReaderCursor( - p, FTS3_SEGCURSOR_ALL_TERM, zTerm, nTerm, isPrefix, 0, pSegcsr); + if( isPrefix ){ + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + break; + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor( + p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); } - for(i=0; rc==SQLITE_OK && inSegment; i++){ rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); } @@ -3340,7 +3416,6 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ assert( p->inTransaction!=0 ); TESTONLY( p->inTransaction = 0 ); TESTONLY( p->mxSavepoint = -1; ); - sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab); return SQLITE_OK; } @@ -3715,7 +3790,6 @@ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); sqlite3Fts3PendingTermsClear(p); - sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab); return SQLITE_OK; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 785a6aa655..00e1c37d2b 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -23,8 +23,6 @@ #include "fts3_tokenizer.h" #include "fts3_hash.h" -#define FTS3_MAX_PREFIX 8 - /* ** This constant controls how often segments are merged. Once there are ** FTS3_MERGE_COUNT segments of level N, they are merged into a single @@ -56,18 +54,22 @@ #define FTS3_VARINT_MAX 10 /* -** FTS4 virtual tables may maintain two separate indexes. One that indexes -** all document terms (the same index FTS3 tables maintain) and another used -** for prefixes. B+-trees that are part of the prefix index have values for -** the %_segdir.level column that are equal to or greater than the following -** value. +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. ** -** It is considered impossible for the regular index to use levels this large. -** In theory it could, but that would require that at least 2^1024 separate -** write operations to be made within the lifetime of the database. +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. */ -#define FTS3_SEGDIR_PREFIXLEVEL 1024 -#define FTS3_SEGDIR_PREFIXLEVEL_STR "1024" +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" /* ** The testcase() macro is only used by the amalgamation. If undefined, @@ -172,23 +174,32 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ - u8 bPrefix; /* True if there is a prefix index */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - /* The following hash table is used to buffer pending index updates during + /* TODO: Fix the first paragraph of this comment. + ** + ** The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. ** When nPendingData exceeds nMaxPendingData, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. */ - int nMaxPendingData; - int nPendingData; - sqlite_int64 iPrevDocid; - Fts3Hash pendingTerms; - Fts3Hash pendingPrefixes; + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ #if defined(SQLITE_DEBUG) /* State variables used for validating that the transaction control @@ -336,10 +347,10 @@ int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending( - Fts3Table*,const char*,int,int,int,Fts3SegReader**); + Fts3Table*,int,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); -int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); @@ -355,15 +366,13 @@ void sqlite3Fts3SegmentsClose(Fts3Table *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_PENDING_PREFIX -2 -#define FTS3_SEGCURSOR_ALL_PREFIX -3 -#define FTS3_SEGCURSOR_ALL_TERM -4 +#define FTS3_SEGCURSOR_ALL -2 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); int sqlite3Fts3SegReaderCursor( - Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *); + Fts3Table *, int, int, const char *, int, int, int, Fts3SegReaderCursor *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index cb91cda347..5c7c7b0b0f 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -96,6 +96,7 @@ static int fts3auxConnectMethod( p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); @@ -375,7 +376,7 @@ static int fts3auxFilterMethod( if( pCsr->zStop==0 ) return SQLITE_NOMEM; } - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL_TERM, + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index ca46269689..d7aa66855a 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -27,7 +27,7 @@ typedef struct Fts3termCursor Fts3termCursor; struct Fts3termTable { sqlite3_vtab base; /* Base class used by SQLite core */ - int bPrefix; /* True for an fts4prefix table */ + int iIndex; /* Index for Fts3Table.aIndex[] */ Fts3Table *pFts3Tab; }; @@ -70,6 +70,12 @@ static int fts3termConnectMethod( int nByte; /* Bytes of space to allocate here */ int rc; /* value returned by declare_vtab() */ Fts3termTable *p; /* Virtual table object to return */ + int iIndex = 0; + + if( argc==5 ){ + iIndex = atoi(argv[4]); + argc--; + } /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ @@ -96,7 +102,8 @@ static int fts3termConnectMethod( p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; - p->bPrefix = (int)pCtx; + p->pFts3Tab->nIndex = iIndex+1; + p->iIndex = iIndex; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); @@ -263,8 +270,7 @@ static int fts3termFilterMethod( pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - rc = sqlite3Fts3SegReaderCursor(pFts3, - p->bPrefix ? FTS3_SEGCURSOR_ALL_PREFIX : FTS3_SEGCURSOR_ALL_TERM, + rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr ); if( rc==SQLITE_OK ){ @@ -355,9 +361,6 @@ int sqlite3Fts3InitTerm(sqlite3 *db){ int rc; /* Return code */ rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_module(db, "fts4prefix", &fts3term_module, (void*)1); - } return rc; } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 5ff7e0b1aa..9a136c3a65 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -183,10 +183,10 @@ struct SegmentNode { #define SQL_NEXT_SEGMENTS_ID 10 #define SQL_INSERT_SEGDIR 11 #define SQL_SELECT_LEVEL 12 -#define SQL_SELECT_ALL_LEVEL 13 +#define SQL_SELECT_LEVEL_RANGE 13 #define SQL_SELECT_LEVEL_COUNT 14 #define SQL_SELECT_SEGDIR_MAX_LEVEL 15 -#define SQL_DELETE_SEGDIR_BY_LEVEL 16 +#define SQL_DELETE_SEGDIR_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 #define SQL_DELETE_DOCSIZE 19 @@ -194,10 +194,11 @@ struct SegmentNode { #define SQL_SELECT_DOCSIZE 21 #define SQL_SELECT_DOCTOTAL 22 #define SQL_REPLACE_DOCTOTAL 23 -#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 #define SQL_DELETE_ALL_TERMS_SEGDIR 25 -#define SQL_DELETE_ALL_PREFIX_SEGDIR 26 + +#define SQL_DELETE_SEGDIR_RANGE 26 /* ** This function is used to obtain an SQLite prepared statement handle @@ -234,12 +235,11 @@ static int fts3SqlStmt( /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' WHERE level < " FTS3_SEGDIR_PREFIXLEVEL_STR - " ORDER BY level DESC, idx ASC", + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1)*" - FTS3_SEGDIR_PREFIXLEVEL_STR, +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", @@ -249,11 +249,11 @@ static int fts3SqlStmt( /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", -/* 24 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' WHERE level >= " FTS3_SEGDIR_PREFIXLEVEL_STR - " ORDER BY level DESC, idx ASC", -/* 25 */ "DELETE FROM %Q.'%q_segdir' WHERE level<" FTS3_SEGDIR_PREFIXLEVEL_STR, -/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level>=" FTS3_SEGDIR_PREFIXLEVEL_STR, +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -409,17 +409,32 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ ** 3: end_block ** 4: root */ -int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ +int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ int rc; sqlite3_stmt *pStmt = 0; - if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ - rc = fts3SqlStmt(p, SQL_SELECT_ALL_PREFIX_LEVEL, &pStmt, 0); - }else if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){ - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + + if( iLevel==FTS3_SEGCURSOR_ALL ){ + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); + } }else{ - assert( iLevel>=0 ); + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); - if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); + } } *ppStmt = pStmt; return rc; @@ -565,9 +580,6 @@ static int fts3PendingTermsAddOne( return rc; } - - - /* ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in @@ -616,6 +628,7 @@ static int fts3PendingTermsAdd( while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ + int i; if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. @@ -626,12 +639,19 @@ static int fts3PendingTermsAdd( break; } - rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingTerms,zToken,nToken); - if( p->bPrefix ){ - int n = (nToken > FTS3_MAX_PREFIX ? FTS3_MAX_PREFIX : nToken); - for(; n>0 && rc==SQLITE_OK; n--){ - rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingPrefixes,zToken,n); - } + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); } } @@ -661,28 +681,21 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ } /* -** Discard the contents of the pending-terms hash table. +** Discard the contents of the pending-terms hash tables. */ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ - Fts3HashElem *pElem; - for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + sqlite3_free(fts3HashData(pElem)); + } + fts3HashClear(pHash); } - fts3HashClear(&p->pendingTerms); p->nPendingData = 0; } -/* -** Discard the contents of the pending-prefixes hash table. -*/ -void sqlite3Fts3PendingPrefixesClear(Fts3Table *p){ - Fts3HashElem *pElem; - for(pElem=fts3HashFirst(&p->pendingPrefixes); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); - } - fts3HashClear(&p->pendingPrefixes); -} - /* ** This function is called by the xUpdate() method as part of an INSERT ** operation. It adds entries for each term in the new record to the @@ -780,7 +793,6 @@ static int fts3DeleteAll(Fts3Table *p){ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); - sqlite3Fts3PendingPrefixesClear(p); /* Delete everything from the %_content, %_segments and %_segdir tables. */ fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); @@ -836,7 +848,7 @@ static void fts3DeleteTerms( ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ -static int fts3SegmentMerge(Fts3Table *, int); +static int fts3SegmentMerge(Fts3Table *, int, int); /* ** This function allocates a new level iLevel index in the segdir table. @@ -853,7 +865,12 @@ static int fts3SegmentMerge(Fts3Table *, int); ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK ** returned. Otherwise, an SQLite error code is returned. */ -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ int rc; /* Return Code */ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext = 0; /* Result of query pNextIdx */ @@ -861,7 +878,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pNextIdx, 1, iLevel); + sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int(pNextIdx, 0); } @@ -875,7 +892,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - rc = fts3SegmentMerge(p, iLevel); + rc = fts3SegmentMerge(p, iIndex, iLevel); *piIdx = 0; }else{ *piIdx = iNext; @@ -1309,23 +1326,22 @@ static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ */ int sqlite3Fts3SegReaderPending( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ const char *zTerm, /* Term to search for */ int nTerm, /* Size of buffer zTerm */ - int isMultiTerm, /* True to visit multiple terms */ - int isPrefixIter, /* 0->pendingTerms, 1->pendingPrefixes */ + int bPrefix, /* True for a prefix iterator */ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ int nElem = 0; /* Size of array at aElem */ int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; - if( isMultiTerm ){ + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ int nAlloc = 0; /* Size of allocated array at aElem */ Fts3HashElem *pE = 0; /* Iterator variable */ - Fts3Hash *pHash; - - pHash = (isPrefixIter ? &p->pendingPrefixes : &p->pendingTerms); for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); @@ -1360,7 +1376,7 @@ int sqlite3Fts3SegReaderPending( }else{ /* The query is a simple term lookup that matches at most one term in ** the index. All that is required is a straight hash-lookup. */ - Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); + Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm); if( pE ){ aElem = &pE; nElem = 1; @@ -1380,7 +1396,7 @@ int sqlite3Fts3SegReaderPending( } } - if( isMultiTerm ){ + if( bPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; @@ -1992,28 +2008,28 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ } /* -** Set *pnMax to the largest segment level in the database for either the -** terms index (if parameter bPrefixIndex is 0) or the prefixes index (if -** parameter bPrefixIndex is 1). +** Set *pnMax to the largest segment level in the database for the index +** iIndex. ** ** Segment levels are stored in the 'level' column of the %_segdir table. ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentMaxLevel(Fts3Table *p, int bPrefixIndex, int *pnMax){ +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ sqlite3_stmt *pStmt; int rc; - assert( bPrefixIndex==0 || bPrefixIndex==1 ); + assert( iIndex>=0 && iIndexnIndex ); /* Set pStmt to the compiled version of: ** - ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1) * 1024 + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? ** ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int(pStmt, 1, bPrefixIndex); + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int(pStmt, 0); } @@ -2036,6 +2052,7 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int bPrefixIndex, int *pnMax){ */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ @@ -2058,27 +2075,23 @@ static int fts3DeleteSegdir( return rc; } - assert( iLevel>=0 - || iLevel==FTS3_SEGCURSOR_ALL_TERM - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX - ); - if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_TERMS_SEGDIR, 0); - }else if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_PREFIX_SEGDIR, 0); - }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){ - sqlite3Fts3PendingPrefixesClear(p); - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - sqlite3Fts3PendingTermsClear(p); - }else if( iLevel>=0 ){ - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); + assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); + if( iLevel==FTS3_SEGCURSOR_ALL ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iLevel); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); } return rc; @@ -2323,46 +2336,52 @@ void sqlite3Fts3SegReaderFinish( ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ -static int fts3SegmentMerge(Fts3Table *p, int iLevel){ +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ - int iNewLevel = 0; /* Level to create new segment at */ + int iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ - rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; - if( iLevel==FTS3_SEGCURSOR_ALL_TERM || iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ + if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically - ** greatest segment level currently present in the database. The index - ** of the new segment is always 0. */ + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iLevel==FTS3_SEGCURSOR_ALL_PREFIX, &iNewLevel); + rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); bIgnoreEmpty = 1; + + }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; + rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = 0; - }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){ - iNewLevel = FTS3_SEGDIR_PREFIXLEVEL; - }else{ - iNewLevel = iLevel+1; - } - rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); + rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); - assert( iNewLevel>=0 ); + assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); + assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; @@ -2378,8 +2397,10 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ if( rc!=SQLITE_OK ) goto finished; assert( pWriter ); - rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); - if( rc!=SQLITE_OK ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment); + if( rc!=SQLITE_OK ) goto finished; + } rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: @@ -2390,19 +2411,16 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ /* -** Flush the contents of pendingTerms to a level 0 segment. +** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; - if( p->bPrefix ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING_PREFIX); - } - if( rc==SQLITE_OK || rc==SQLITE_DONE ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); - } - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; + int i; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; } + sqlite3Fts3PendingTermsClear(p); return rc; } @@ -2554,6 +2572,23 @@ static void fts3UpdateDocTotals( sqlite3_free(a); } +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int i; + int bSeenDone = 0; + int rc = SQLITE_OK; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -2570,10 +2605,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX); - if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM); - } + rc = fts3DoOptimize(p, 0); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -2586,9 +2618,6 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = SQLITE_ERROR; } - sqlite3Fts3SegmentsClose(p); - sqlite3Fts3PendingTermsClear(p); - sqlite3Fts3PendingPrefixesClear(p); return rc; } @@ -2905,30 +2934,19 @@ int sqlite3Fts3UpdateMethod( */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; - int bReturnDone = 0; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX); - if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM); - } - if( rc==SQLITE_DONE ){ - bReturnDone = 1; - rc = SQLITE_OK; - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); - if( rc==SQLITE_OK ){ - sqlite3Fts3PendingTermsClear(p); - sqlite3Fts3PendingPrefixesClear(p); - } + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); } } sqlite3Fts3SegmentsClose(p); - return ((rc==SQLITE_OK && bReturnDone) ? SQLITE_DONE : rc); + return rc; } #endif diff --git a/manifest b/manifest index bf93a84328..aff0cdf712 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\sfts4\soption\sprefix=1\sis\sspecified,\shave\sthe\sfts4\smodule\smaintain\san\sindex\sof\sprefixes\sas\swell\sas\sterms. -D 2011-05-24T18:49:45.786 +C Change\sfts4\sso\sthat\sthe\sprefix=\sparameter\sis\spasses\sa\scomma-separated\slist\sof\sintegers.\sFor\seach\sinteger\sN,\sa\sseparate\sindex\sof\sall\sprefixes\sof\slength\sN\sbytes\sis\screated. +D 2011-05-25T18:34:53.807 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 4a48bfef342badba0a71bdeb5354edaa3ad83382 +F ext/fts3/fts3.c 9ebda4cd406e5aa234977caa97c2045d36048753 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 02699211c0b6cf5aa713cc68c527c9a6e9159fbe -F ext/fts3/fts3_aux.c d68d8e4d39e0342302d2c834618755af7c8058ea +F ext/fts3/fts3Int.h deeeac21d17da06683a79b40ae119b93cf86f90a +F ext/fts3/fts3_aux.c 2817a1ec9ffbad673cb1a1527ad807811bc7645b F ext/fts3/fts3_expr.c 5f49e0deaf723724b08100bb3ff40aab02ad0c93 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 F ext/fts3/fts3_snippet.c 92b40397b28422c35c4127492d7ac6da34d1966a -F ext/fts3/fts3_term.c cd226a311940b8ef414d5c1f7c74971a47cacedb +F ext/fts3/fts3_term.c 0ade1812c0e97f394b58299810dfd5d2fb7ba501 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c 0fd6a55c774731852f889007fc6edb1b99819ee5 +F ext/fts3/fts3_write.c 416b367f56f3100314fa76615535178718c645e1 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -473,7 +473,7 @@ F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 -F test/fts3prefix.test 5b4e08c63d5d4a79e54754dc6b2209b03c885200 +F test/fts3prefix.test c51b04f38ce232cd0c9edd3cc60dc981db5e2726 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 @@ -939,10 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 651ef24249d8c22c4f13e4c0bb98a60099cfd23a -R c499b030bb5d74c4afca730dec7a05bc -T *branch * fts3-prefix-search -T *sym-fts3-prefix-search * -T -sym-trunk * +P b5bdc639898ee22eebedeb560810e94e74de8aa4 +R e9f427c5bc714eae400763155f710174 U dan -Z 0c60a0f2885eb6df2e24e22407faffbe +Z 6f0f673a9e9eaf0f72d887565c9f2227 diff --git a/manifest.uuid b/manifest.uuid index 747bc2e8b4..3ae4c6e5e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5bdc639898ee22eebedeb560810e94e74de8aa4 \ No newline at end of file +be59bf49402d2e2f4b95fb6668849f3745cb7bf2 \ No newline at end of file diff --git a/test/fts3prefix.test b/test/fts3prefix.test index 95a5c89629..2e0c7a8ede 100644 --- a/test/fts3prefix.test +++ b/test/fts3prefix.test @@ -16,56 +16,75 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts3prefix +ifcapable !fts3 { + finish_test + return +} + # This proc tests that the prefixes index appears to represent the same content # as the terms index. # -proc fts3_terms_and_prefixes {db tbl} { - $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl);" - $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4prefix($tbl);" +proc fts3_terms_and_prefixes {db tbl prefixlengths} { - $db eval { - CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; - CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; - CREATE INDEX temp.idx ON prefixes(term); - DROP TABLE fts3check1; - DROP TABLE fts3check2; - } + set iIndex 0 + foreach len $prefixlengths { + incr iIndex + $db eval { + DROP TABLE IF EXISTS fts3check1; + DROP TABLE IF EXISTS fts3check2; + } + $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl, 0);" + $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4term($tbl, $iIndex);" - $db eval { SELECT term, docid, col, pos FROM temp.terms } a { - set nMax [expr [string length $a(term)] - 1] - if {$nMax>8} {set nMax 8} - for {set n 0} {$n < $nMax} {incr n} { - set t [string range $a(term) 0 $n] + $db eval { + DROP TABLE IF EXISTS temp.terms; + DROP TABLE IF EXISTS temp.prefixes; + CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; + CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; + CREATE INDEX temp.idx ON prefixes(term); + DROP TABLE fts3check1; + DROP TABLE fts3check2; + } + + set nExpect 0 + $db eval { SELECT term, docid, col, pos FROM temp.terms } a { + if {[string length $a(term)]<$len} continue + incr nExpect + set prefix [string range $a(term) 0 [expr $len-1]] set r [$db one { SELECT count(*) FROM temp.prefixes WHERE - term = $t AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) + term = $prefix AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) }] if {$r != 1} { error "$t, $a(docid), $a(col), $a(pos)" } } - } - execsql { DROP TABLE temp.prefixes } - execsql { DROP TABLE temp.terms } + set nCount [$db one {SELECT count(*) FROM temp.prefixes}] + if {$nCount != $nExpect} { + error "prefixes.count(*) is $nCount expected $nExpect" + } + + execsql { DROP TABLE temp.prefixes } + execsql { DROP TABLE temp.terms } - set terms_layout [$db eval " - SELECT level, idx FROM ${tbl}_segdir WHERE level < 1024 ORDER by 1, 2 - "] - set prefixes_layout [$db eval " - SELECT level-1024, idx FROM ${tbl}_segdir WHERE level >= 1024 ORDER by 1, 2 - "] + set list [list] + $db eval " + SELECT sum( 1 << (16*(level%1024)) ) AS total, (level/1024) AS tree + FROM ${tbl}_segdir GROUP BY tree + " { + lappend list [list $total $tree] + } - if {$terms_layout != $prefixes_layout} { - puts "TERMS LAYOUT: $terms_layout" - puts "PREFIX LAYOUT: $prefixes_layout" - error "Terms and prefixes are comprised of different b-trees" + if { [lsort -integer -index 0 $list] != [lsort -integer -index 1 $list] } { + error "inconsistent tree structures: $list" + } } return "" } -proc fts3_tap_test {tn db tbl} { - uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl] ""] +proc fts3_tap_test {tn db tbl lens} { + uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl $lens] ""] } #------------------------------------------------------------------------- @@ -73,21 +92,24 @@ proc fts3_tap_test {tn db tbl} { # being constructed correctly for the simplest possible case. # do_execsql_test 1.1 { - CREATE VIRTUAL TABLE t1 USING fts4(prefix=1); - CREATE VIRTUAL TABLE prefixes USING fts4prefix(t1); + CREATE VIRTUAL TABLE t1 USING fts4(prefix='1,3,6'); + + CREATE VIRTUAL TABLE p1 USING fts4term(t1, 1); + CREATE VIRTUAL TABLE p2 USING fts4term(t1, 2); + CREATE VIRTUAL TABLE p3 USING fts4term(t1, 3); CREATE VIRTUAL TABLE terms USING fts4term(t1); } do_execsql_test 1.2 { INSERT INTO t1 VALUES('sqlite mysql firebird'); } -do_execsql_test 1.3 { - SELECT term FROM prefixes; -} {f fi fir fire fireb firebi firebir firebird m my mys mysq mysql s sq sql sqli sqlit sqlite} +do_execsql_test 1.3.1 { SELECT term FROM p1 } {f m s} +do_execsql_test 1.3.2 { SELECT term FROM p2 } {fir mys sql} +do_execsql_test 1.3.3 { SELECT term FROM p3 } {firebi sqlite} do_execsql_test 1.4 { SELECT term FROM terms; } {firebird mysql sqlite} -fts3_tap_test 1.5 db t1 +fts3_tap_test 1.5 db t1 {1 3 6} #------------------------------------------------------------------------- # A slightly more complicated dataset. This test also verifies that DELETE @@ -115,15 +137,15 @@ do_execsql_test 2.1 { INSERT INTO t1 VALUES('Google. It is now developed and maintained as part'); INSERT INTO t1 VALUES('of SQLite. '); } -fts3_tap_test 2.2 db t1 +fts3_tap_test 2.2 db t1 {1 3 6} do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; } -fts3_tap_test 2.4 db t1 +fts3_tap_test 2.4 db t1 {1 3 6} do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') } -fts3_tap_test 2.6 db t1 +fts3_tap_test 2.6 db t1 {1 3 6} do_execsql_test 3.1 { - CREATE VIRTUAL TABLE t2 USING fts4(prefix=1); + CREATE VIRTUAL TABLE t2 USING fts4(prefix='1,2,3'); INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); INSERT INTO t2 VALUES('fleet was forced to take shelter at'); @@ -148,16 +170,16 @@ do_execsql_test 3.1 { INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); } -fts3_tap_test 3.2 db t2 +fts3_tap_test 3.2 db t2 {1 2 3} do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}} -fts3_tap_test 3.4 db t2 +fts3_tap_test 3.4 db t2 {1 2 3} #------------------------------------------------------------------------- # Simple tests for reading the prefix-index. # do_execsql_test 4.1 { - CREATE VIRTUAL TABLE t3 USING fts4(prefix=1); + CREATE VIRTUAL TABLE t3 USING fts4(prefix="1,4"); INSERT INTO t3 VALUES('one two three'); INSERT INTO t3 VALUES('four five six'); INSERT INTO t3 VALUES('seven eight nine'); From 98b08e71f28a0ba5a365e927c7423d79c2b733f8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 25 May 2011 19:17:32 +0000 Subject: [PATCH 06/69] If a prefix index of size N is not present, use a prefix index of size N+1 along with the terms index for queries for prefixes of length N. FossilOrigin-Name: cc83991caae7c7d647432d5711b6cd80228c3002 --- ext/fts3/fts3.c | 86 +++++++++++++++++++++++++++++-------------- ext/fts3/fts3_write.c | 2 +- manifest | 16 ++++---- manifest.uuid | 2 +- test/fts3prefix.test | 6 +++ 5 files changed, 75 insertions(+), 37 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 26239be95d..541d6c539d 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2237,11 +2237,7 @@ static int fts3SegReaderCursorAppend( return SQLITE_OK; } -/* -** Set up a cursor object for iterating through a full-text index or a -** single level therein. -*/ -int sqlite3Fts3SegReaderCursor( +static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ @@ -2253,25 +2249,8 @@ int sqlite3Fts3SegReaderCursor( ){ int rc = SQLITE_OK; int rc2; - int iAge = 0; sqlite3_stmt *pStmt = 0; - assert( iIndex>=0 && iIndexnIndex ); - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( iLevelaIndex==0 ); - - memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); - /* If iLevel is less than 0 and this is not a scan, include a seg-reader ** for the pending-terms. If this is a scan, then this call must be being ** made by an fts4aux module, not an FTS table. In this case calling @@ -2310,12 +2289,11 @@ int sqlite3Fts3SegReaderCursor( if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; } - rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pSeg + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg ); if( rc!=SQLITE_OK ) goto finished; rc = fts3SegReaderCursorAppend(pCsr, pSeg); - iAge++; } } @@ -2326,6 +2304,49 @@ int sqlite3Fts3SegReaderCursor( return rc; } +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3SegReaderCursor *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelaIndex==0 ); + + memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); + + return fts3SegReaderCursor( + p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr + ); +} + +static int fts3SegReaderCursorAddZero( + Fts3Table *p, + const char *zTerm, + int nTerm, + Fts3SegReaderCursor *pCsr +){ + return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); +} + static int fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ @@ -2345,12 +2366,23 @@ static int fts3TermSegReaderCursor( Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; if( isPrefix ){ - for(i=1; inIndex; i++){ + for(i=1; bFound==0 && inIndex; i++){ if( p->aIndex[i].nPrefix==nTerm ){ bFound = 1; rc = sqlite3Fts3SegReaderCursor( p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); - break; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr); + } } } } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 9a136c3a65..3d4e022298 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -422,7 +422,7 @@ int sqlite3Fts3AllSegdirs( assert( iLevel=0 && iIndexnIndex ); - if( iLevel==FTS3_SEGCURSOR_ALL ){ + if( iLevel<0 ){ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); if( rc==SQLITE_OK ){ diff --git a/manifest b/manifest index d0a1e83625..7cd9a1f2c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sexperimental\sfts3-prefix-search\sbranch. -D 2011-05-25T18:47:26.259 +C If\sa\sprefix\sindex\sof\ssize\sN\sis\snot\spresent,\suse\sa\sprefix\sindex\sof\ssize\sN+1\salong\swith\sthe\sterms\sindex\sfor\squeries\sfor\sprefixes\sof\slength\sN. +D 2011-05-25T19:17:32.713 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 925360f6c043fb87719e56520470265d69c5f870 +F ext/fts3/fts3.c b44e0abc7aaef8d5489533b3f0556b28097378f9 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h deeeac21d17da06683a79b40ae119b93cf86f90a F ext/fts3/fts3_aux.c 2817a1ec9ffbad673cb1a1527ad807811bc7645b @@ -75,7 +75,7 @@ F ext/fts3/fts3_term.c 0ade1812c0e97f394b58299810dfd5d2fb7ba501 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c 416b367f56f3100314fa76615535178718c645e1 +F ext/fts3/fts3_write.c beaa93bcbe1664fcada75b70893f9b221acf2777 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -473,7 +473,7 @@ F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 -F test/fts3prefix.test c51b04f38ce232cd0c9edd3cc60dc981db5e2726 +F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P be59bf49402d2e2f4b95fb6668849f3745cb7bf2 d807304a695fc85402b86e1cd32a6e3bbb2823c8 -R 2e09e5d0236bf18c24c777dcd7acf739 +P f0f0a03db214b68a37069f64c27ae8520220c900 +R 95d28d5ab6425c93022acec043aee69b U dan -Z fd8764569a495dba32fc732cd25f7598 +Z 2f1dd28b688335ffa6210dbb33afadf5 diff --git a/manifest.uuid b/manifest.uuid index 00ceff796f..3079457335 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f0f0a03db214b68a37069f64c27ae8520220c900 \ No newline at end of file +cc83991caae7c7d647432d5711b6cd80228c3002 \ No newline at end of file diff --git a/test/fts3prefix.test b/test/fts3prefix.test index 2e0c7a8ede..f5e31f3208 100644 --- a/test/fts3prefix.test +++ b/test/fts3prefix.test @@ -193,5 +193,11 @@ do_execsql_test 4.3 { do_execsql_test 4.4 { SELECT * FROM t3 WHERE t3 MATCH 's*' } {{four five six} {seven eight nine}} +do_execsql_test 4.5 { + SELECT * FROM t3 WHERE t3 MATCH 'sev*' +} {{seven eight nine}} +do_execsql_test 4.6 { + SELECT * FROM t3 WHERE t3 MATCH 'one*' +} {{one two three}} finish_test From 382874fc5c4e79c5e7ea7b26854afd2e5195db2a Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 28 May 2011 15:57:40 +0000 Subject: [PATCH 07/69] Minor changes made while planning a larger change. FossilOrigin-Name: 84097a4c759b1d65890af885f137d3cb16eef584 --- ext/fts3/fts3.c | 71 +++++++++++++---------- ext/fts3/fts3Int.h | 36 ++++++------ ext/fts3/fts3_expr.c | 124 ++++++++++++++++++++++++---------------- ext/fts3/fts3_snippet.c | 18 +++--- ext/fts3/fts3_write.c | 16 +++--- manifest | 22 +++---- manifest.uuid | 2 +- test/hook.test | 28 +++++++++ 8 files changed, 191 insertions(+), 126 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 541d6c539d..536c47fc78 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2767,11 +2767,13 @@ static int fts3NearMerge( ** each doclist that are not within nNear tokens of a corresponding entry ** in the other doclist. */ -int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ +int sqlite3Fts3ExprNearTrim(Fts3Expr *pELeft, Fts3Expr *pERight, int nNear){ int rc; /* Return code */ + Fts3Phrase *pLeft = pELeft->pPhrase; + Fts3Phrase *pRight = pERight->pPhrase; - assert( pLeft->eType==FTSQUERY_PHRASE ); - assert( pRight->eType==FTSQUERY_PHRASE ); + assert( pELeft->eType==FTSQUERY_PHRASE && pLeft ); + assert( pERight->eType==FTSQUERY_PHRASE && pRight ); assert( pLeft->isLoaded && pRight->isLoaded ); if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ @@ -2785,8 +2787,8 @@ int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ int nOut; /* Size of buffer aOut in bytes */ rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, + pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist, + pRight->nToken, pRight->aDoclist, pRight->nDoclist, &aOut, &nOut ); if( rc!=SQLITE_OK ) return rc; @@ -2795,8 +2797,8 @@ int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ pRight->nDoclist = nOut; rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, + pRight->nToken, pRight->aDoclist, pRight->nDoclist, + pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist, &aOut, &nOut ); sqlite3_free(pLeft->aDoclist); @@ -3459,12 +3461,19 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ */ int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ int rc; - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->eType==FTSQUERY_PHRASE && pPhrase ); assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); + rc = fts3EvalExpr(pCsr, pExpr, &pPhrase->aDoclist, &pPhrase->nDoclist, 1); return rc; } +/* +** TODO: This is something to do with matchinfo(). Similar to +** sqlite3ExprLoadDoclists() but slightly different. +** +** UPDATE: Only used when there are deferred tokens. +*/ int sqlite3Fts3ExprLoadFtDoclist( Fts3Cursor *pCsr, Fts3Expr *pExpr, @@ -3510,44 +3519,46 @@ char *sqlite3Fts3FindPositions( sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ int iCol /* Column of requested pos-list */ ){ - assert( pExpr->isLoaded ); - if( pExpr->aDoclist ){ - char *pEnd = &pExpr->aDoclist[pExpr->nDoclist]; + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pPhrase->isLoaded ); + + if( pPhrase->aDoclist ){ + char *pEnd = &pPhrase->aDoclist[pPhrase->nDoclist]; char *pCsr; - if( pExpr->pCurrent==0 ){ + if( pPhrase->pCurrent==0 ){ if( pCursor->desc==0 ){ - pExpr->pCurrent = pExpr->aDoclist; - pExpr->iCurrent = 0; - fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent); + pPhrase->pCurrent = pPhrase->aDoclist; + pPhrase->iCurrent = 0; + fts3GetDeltaVarint(&pPhrase->pCurrent, &pPhrase->iCurrent); }else{ - pCsr = pExpr->aDoclist; + pCsr = pPhrase->aDoclist; while( pCsriCurrent); + fts3GetDeltaVarint(&pCsr, &pPhrase->iCurrent); fts3PoslistCopy(0, &pCsr); } - fts3ReversePoslist(pExpr->aDoclist, &pCsr); - pExpr->pCurrent = pCsr; + fts3ReversePoslist(pPhrase->aDoclist, &pCsr); + pPhrase->pCurrent = pCsr; } } - pCsr = pExpr->pCurrent; + pCsr = pPhrase->pCurrent; assert( pCsr ); while( (pCursor->desc==0 && pCsrdesc && pCsr>pExpr->aDoclist) + || (pCursor->desc && pCsr>pPhrase->aDoclist) ){ - if( pCursor->desc==0 && pExpr->iCurrentdesc==0 && pPhrase->iCurrentiCurrent); + fts3GetDeltaVarint(&pCsr, &pPhrase->iCurrent); } - pExpr->pCurrent = pCsr; - }else if( pCursor->desc && pExpr->iCurrent>iDocid ){ - fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent); - fts3ReversePoslist(pExpr->aDoclist, &pCsr); - pExpr->pCurrent = pCsr; + pPhrase->pCurrent = pCsr; + }else if( pCursor->desc && pPhrase->iCurrent>iDocid ){ + fts3GetReverseDeltaVarint(&pCsr, pPhrase->aDoclist, &pPhrase->iCurrent); + fts3ReversePoslist(pPhrase->aDoclist, &pCsr); + pPhrase->pCurrent = pCsr; }else{ - if( pExpr->iCurrent==iDocid ){ + if( pPhrase->iCurrent==iDocid ){ int iThis = 0; if( iCol<0 ){ /* If iCol is negative, return a pointer to the start of the diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 00e1c37d2b..42be10773a 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -268,17 +268,16 @@ struct Fts3Cursor { ** sequence. A single token is the base case and the most common case. ** For a sequence of tokens contained in double-quotes (i.e. "one two three") ** nToken will be the number of tokens in the string. -** -** The nDocMatch and nMatch variables contain data that may be used by the -** matchinfo() function. They are populated when the full-text index is -** queried for hits on the phrase. If one or more tokens in the phrase -** are deferred, the nDocMatch and nMatch variables are populated based -** on the assumption that the */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + int bFulltext; /* True if full-text index was used */ Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ @@ -288,18 +287,24 @@ struct Fts3Phrase { /* Variables populated by fts3_expr.c when parsing a MATCH expression */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ - int isNot; /* Phrase prefixed by unary not (-) operator */ + + int isLoaded; /* True if aDoclist/nDoclist are initialized. */ + char *aDoclist; /* Buffer containing doclist */ + int nDoclist; /* Size of aDoclist in bytes */ + sqlite3_int64 iCurrent; + char *pCurrent; + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ }; /* ** A tree of these objects forms the RHS of a MATCH operator. ** -** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded -** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, -** containing the results of the NEAR or phrase query in FTS3 doclist -** format. As usual, the initial "Length" field found in doclists stored -** on disk is omitted from this buffer. +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. ** ** Variable pCurrent always points to the start of a docid field within ** aDoclist. Since the doclist is usually scanned in docid order, this can @@ -312,13 +317,6 @@ struct Fts3Expr { Fts3Expr *pLeft; /* Left operand */ Fts3Expr *pRight; /* Right operand */ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ - - int isLoaded; /* True if aDoclist/nDoclist are initialized. */ - char *aDoclist; /* Buffer containing doclist */ - int nDoclist; /* Size of aDoclist in bytes */ - - sqlite3_int64 iCurrent; - char *pCurrent; }; /* diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 43f6d84a84..095841d142 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -81,12 +81,21 @@ int sqlite3_fts3_enable_parentheses = 0; #include #include +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ sqlite3_context *pCtx; /* Write error message here */ int nNest; /* Number of nested brackets */ }; @@ -172,7 +181,7 @@ static int getNextToken( iEnd++; } if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pRet->pPhrase->isNot = 1; + pParse->isNot = 1; } } nConsumed = iEnd; @@ -224,36 +233,55 @@ static int getNextString( char *zTemp = 0; int nTemp = 0; + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); if( rc==SQLITE_OK ){ int ii; pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ - const char *zToken; - int nToken, iBegin, iEnd, iPos; - rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); + const char *zByte; + int nByte, iBegin, iEnd, iPos; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ - int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); - zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); - if( !p || !zTemp ){ - goto no_mem; - } - if( ii==0 ){ - memset(p, 0, nByte); - p->pPhrase = (Fts3Phrase *)&p[1]; - } - p->pPhrase = (Fts3Phrase *)&p[1]; - memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); - p->pPhrase->nToken = ii+1; - p->pPhrase->aToken[ii].n = nToken; - memcpy(&zTemp[nTemp], zToken, nToken); - nTemp += nToken; - if( iEndpPhrase->aToken[ii].isPrefix = 1; - }else{ - p->pPhrase->aToken[ii].isPrefix = 0; - } + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndpPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); - p = fts3ReallocOrFree(p, nByte + nTemp); - if( !p ){ - goto no_mem; - } - if( zTemp ){ - zNew = &(((char *)p)[nByte]); - memcpy(zNew, zTemp, nTemp); - }else{ - memset(p, 0, nByte+nTemp); - } - p->pPhrase = (Fts3Phrase *)&p[1]; - for(jj=0; jjpPhrase->nToken; jj++){ - p->pPhrase->aToken[jj].z = &zNew[nNew]; - nNew += p->pPhrase->aToken[jj].n; - } - sqlite3_free(zTemp); + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } rc = SQLITE_OK; } @@ -341,6 +365,8 @@ static int getNextNode( const char *zInput = z; int nInput = n; + pParse->isNot = 0; + /* Skip over any whitespace before checking for a keyword, an open or ** close bracket, or a quoted string. */ @@ -559,7 +585,7 @@ static int fts3ExprParse( int isPhrase; if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot + && p->eType==FTSQUERY_PHRASE && pParse->isNot ){ /* Create an implicit NOT operator. */ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); @@ -577,7 +603,6 @@ static int fts3ExprParse( p = pPrev; }else{ int eType = p->eType; - assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot ); isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); /* The isRequirePhrase variable is set to true if a phrase or @@ -740,9 +765,10 @@ int sqlite3Fts3ExprParse( */ void sqlite3Fts3ExprFree(Fts3Expr *p){ if( p ){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); - sqlite3_free(p->aDoclist); + if( p->pPhrase ) sqlite3_free(p->pPhrase->aDoclist); sqlite3_free(p); } } @@ -800,7 +826,7 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; zBuf = sqlite3_mprintf( - "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot); + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); for(i=0; zBuf && inToken; i++){ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, pPhrase->aToken[i].n, pPhrase->aToken[i].z, diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 8bf9ed2d5a..b31a1cf359 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -228,16 +228,17 @@ static int fts3ExprNearTrim(Fts3Expr *pExpr){ */ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; UNUSED_PARAMETER(iPhrase); p->nPhrase++; - p->nToken += pExpr->pPhrase->nToken; + p->nToken += pPhrase->nToken; - if( pExpr->isLoaded==0 ){ + if( pPhrase->isLoaded==0 ){ rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); - pExpr->isLoaded = 1; + pPhrase->isLoaded = 1; if( rc==SQLITE_OK ){ rc = fts3ExprNearTrim(pExpr); } @@ -826,16 +827,15 @@ static int fts3ExprGlobalHitsCb( ){ MatchInfo *p = (MatchInfo *)pCtx; Fts3Cursor *pCsr = p->pCursor; + Fts3Phrase *pPhrase = pExpr->pPhrase; char *pIter; char *pEnd; char *pFree = 0; u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; - assert( pExpr->isLoaded ); - assert( pExpr->eType==FTSQUERY_PHRASE ); + assert( pPhrase->isLoaded ); if( pCsr->pDeferred ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; for(ii=0; iinToken; ii++){ if( pPhrase->aToken[ii].bFulltext ) break; @@ -855,8 +855,8 @@ static int fts3ExprGlobalHitsCb( return SQLITE_OK; } }else{ - pIter = pExpr->aDoclist; - pEnd = &pExpr->aDoclist[pExpr->nDoclist]; + pIter = pPhrase->aDoclist; + pEnd = &pPhrase->aDoclist[pPhrase->nDoclist]; } /* Fill in the global hit count matrix row for this phrase. */ @@ -885,7 +885,7 @@ static int fts3ExprLocalHitsCb( for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; - if( pExpr->aDoclist ){ + if( pExpr->pPhrase->aDoclist ){ char *pCsr; pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 3d4e022298..f664dec8cf 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -2642,15 +2642,17 @@ char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ */ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; fts3DeferredDoclistClear(pExpr->pLeft); fts3DeferredDoclistClear(pExpr->pRight); - if( pExpr->isLoaded ){ - sqlite3_free(pExpr->aDoclist); - pExpr->isLoaded = 0; - pExpr->aDoclist = 0; - pExpr->nDoclist = 0; - pExpr->pCurrent = 0; - pExpr->iCurrent = 0; + if( pPhrase ){ + assert( pExpr->eType==FTSQUERY_PHRASE ); + sqlite3_free(pPhrase->aDoclist); + pPhrase->isLoaded = 0; + pPhrase->aDoclist = 0; + pPhrase->nDoclist = 0; + pPhrase->pCurrent = 0; + pPhrase->iCurrent = 0; } } } diff --git a/manifest b/manifest index 7cd9a1f2c5..950ee87f2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\sprefix\sindex\sof\ssize\sN\sis\snot\spresent,\suse\sa\sprefix\sindex\sof\ssize\sN+1\salong\swith\sthe\sterms\sindex\sfor\squeries\sfor\sprefixes\sof\slength\sN. -D 2011-05-25T19:17:32.713 +C Minor\schanges\smade\swhile\splanning\sa\slarger\schange. +D 2011-05-28T15:57:40.694 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c b44e0abc7aaef8d5489533b3f0556b28097378f9 +F ext/fts3/fts3.c eb59cdd89e9309ab9b2dca196a7c52f9e8927319 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h deeeac21d17da06683a79b40ae119b93cf86f90a +F ext/fts3/fts3Int.h 832f4d421f03a9d364186e779ee51994df500c62 F ext/fts3/fts3_aux.c 2817a1ec9ffbad673cb1a1527ad807811bc7645b -F ext/fts3/fts3_expr.c 5f49e0deaf723724b08100bb3ff40aab02ad0c93 +F ext/fts3/fts3_expr.c 25e30cf24198333f2ed545af905b168e88f56903 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c 92b40397b28422c35c4127492d7ac6da34d1966a +F ext/fts3/fts3_snippet.c 6ee626017ddcf7d72ca4f587724e3506434fc0d7 F ext/fts3/fts3_term.c 0ade1812c0e97f394b58299810dfd5d2fb7ba501 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c beaa93bcbe1664fcada75b70893f9b221acf2777 +F ext/fts3/fts3_write.c ed58c53fbcbc2ea79258e734159a5951ffeb1bd4 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -489,7 +489,7 @@ F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test dd7001ac86d09c154a7dff064f4739c60e2b312c F test/fuzzer1.test 3105b5a89a6cb0d475f0877debec942fe4143462 -F test/hook.test f04c3412463f8ec117c1c704c74ca0f627ce733a +F test/hook.test f2277c309e4ee8067d95d6b9b315568e9d5329b2 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P f0f0a03db214b68a37069f64c27ae8520220c900 -R 95d28d5ab6425c93022acec043aee69b +P cc83991caae7c7d647432d5711b6cd80228c3002 +R eef52aec1faead4921ebaf39c1605d2b U dan -Z 2f1dd28b688335ffa6210dbb33afadf5 +Z 01d8f7aea2ce8ebb35c05ccc519b614f diff --git a/manifest.uuid b/manifest.uuid index 3079457335..6768a6838c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc83991caae7c7d647432d5711b6cd80228c3002 \ No newline at end of file +84097a4c759b1d65890af885f137d3cb16eef584 \ No newline at end of file diff --git a/test/hook.test b/test/hook.test index 6496d41e13..c4dfb2e4f9 100644 --- a/test/hook.test +++ b/test/hook.test @@ -274,6 +274,34 @@ ifcapable compound&&attach { set ::update_hook } [list] } + +do_test hook-4.4 { + execsql { + CREATE TABLE t4(a UNIQUE, b); + INSERT INTO t4 VALUES(1, 'a'); + INSERT INTO t4 VALUES(2, 'b'); + } + set ::update_hook [list] + execsql { + REPLACE INTO t4 VALUES(1, 'c'); + } + set ::update_hook +} [list INSERT main t4 3 ] +do_execsql_test hook-4.4.1 { + SELECT * FROM t4 ORDER BY a; +} {1 c 2 b} +do_test hook-4.4.2 { + set ::update_hook [list] + execsql { + PRAGMA recursive_triggers = on; + REPLACE INTO t4 VALUES(1, 'd'); + } + set ::update_hook +} [list INSERT main t4 4 ] +do_execsql_test hook-4.4.3 { + SELECT * FROM t4 ORDER BY a; +} {1 d 2 b} + db update_hook {} # #---------------------------------------------------------------------------- From 1fbb230f90e150c3c42c294afc06eb32901193e5 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 30 May 2011 14:35:48 +0000 Subject: [PATCH 08/69] Make sure the P5 argument to the OP_VUpdate opcode is always set to a valid conflict resolution code. FossilOrigin-Name: e3350dbd9f472c27ea6bc872d85098ac89d874c7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/delete.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 701dc47e95..f9e0a1ae94 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\s"#ifdef"\sfrom\scheck-in\s[03f7d36a8a]\sthat\sshould\shave\sbeen\s"#ifndef". -D 2011-05-28T19:24:15.852 +C Make\ssure\sthe\sP5\sargument\sto\sthe\sOP_VUpdate\sopcode\sis\salways\sset\sto\sa\nvalid\sconflict\sresolution\scode. +D 2011-05-30T14:35:48.783 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -130,7 +130,7 @@ F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 F src/date.c 1548fdac51377e4e7833251de878b4058c148e1b -F src/delete.c 7a24fcc9a31664d145acb97ce56b6d9f249a25e4 +F src/delete.c cecc926c70783452f3e8eb452c728291ce1a0b21 F src/expr.c e3cf0957c6b8faaaf7386a3bc69e53c0dc9705be F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c a43ba8a005fb5efd1deeee06853e3a6120d46a91 @@ -938,7 +938,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 5f15579f8ca747f0fa0a7867ceffc8c7b8f904e6 -R a63903941bea8f77658823b8763468e2 +P edb865c35415f9553f8279028120f7b8de2bf7e2 +R 3b765e3c18f9b2a8d8763c389176a0b5 U drh -Z 62e9bfe1cc7669e25feec8cecb39066d +Z 3e70f15cebd9551500dd66c2327a7d64 diff --git a/manifest.uuid b/manifest.uuid index 60ef9e438e..f3911243ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -edb865c35415f9553f8279028120f7b8de2bf7e2 \ No newline at end of file +e3350dbd9f472c27ea6bc872d85098ac89d874c7 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index e5389e2b6a..a2df773af8 100644 --- a/src/delete.c +++ b/src/delete.c @@ -401,6 +401,7 @@ void sqlite3DeleteFrom( const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iRowid, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, OE_Abort); sqlite3MayAbort(pParse); }else #endif From f4b595b929449c03383f8c20d1929f284c960cc9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 30 May 2011 15:06:48 +0000 Subject: [PATCH 09/69] Update evidence marks on the URI filename tests to conform to the latest documentation. FossilOrigin-Name: 1bab03c4811b5e5b3d15632bc2a3844891f9fad7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/e_uri.test | 18 ++++++++++-------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f9e0a1ae94..207bb8d5bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sP5\sargument\sto\sthe\sOP_VUpdate\sopcode\sis\salways\sset\sto\sa\nvalid\sconflict\sresolution\scode. -D 2011-05-30T14:35:48.783 +C Update\sevidence\smarks\son\sthe\sURI\sfilename\stests\sto\sconform\sto\sthe\slatest\ndocumentation. +D 2011-05-30T15:06:48.453 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -380,7 +380,7 @@ F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 F test/e_select.test bf385ae3aa0f014c4933ae66fd3e1302138493eb F test/e_select2.test 5c3d3da19c7b3e90ae444579db2b70098599ab92 F test/e_update.test 963d6876064e65f318d1c93aaed36a02b9b389bf -F test/e_uri.test 9ce11319fb9b271bf7392027f913f7830e93e7a7 +F test/e_uri.test b6da43a10f44d9aa0aff5ffa3c2f3de668361255 F test/e_vacuum.test 6c09c2af7f2f140518f371c5342100118f779dcf F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 6d91a5286f59add0cfcbb2d0da913b76f2242398 @@ -938,7 +938,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P edb865c35415f9553f8279028120f7b8de2bf7e2 -R 3b765e3c18f9b2a8d8763c389176a0b5 +P e3350dbd9f472c27ea6bc872d85098ac89d874c7 +R 31eb4302622f868d251135572f646bae U drh -Z 3e70f15cebd9551500dd66c2327a7d64 +Z 47e43c01cd8d7b7136f7bc52a005ac53 diff --git a/manifest.uuid b/manifest.uuid index f3911243ba..45419984ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3350dbd9f472c27ea6bc872d85098ac89d874c7 \ No newline at end of file +1bab03c4811b5e5b3d15632bc2a3844891f9fad7 \ No newline at end of file diff --git a/test/e_uri.test b/test/e_uri.test index b0e2f876de..3e47427941 100644 --- a/test/e_uri.test +++ b/test/e_uri.test @@ -48,10 +48,11 @@ proc open_uri_error {uri} { # and the filename argument begins with "file:", then the filename is # interpreted as a URI. # -# EVIDENCE-OF: R-00067-59538 URI filename interpretation is enabled if +# EVIDENCE-OF: R-32637-34037 URI filename interpretation is enabled if # the SQLITE_OPEN_URI flag is is set in the fourth argument to # sqlite3_open_v2(), or if it has been enabled globally using the -# SQLITE_CONFIG_URI option with the sqlite3_config() method. +# SQLITE_CONFIG_URI option with the sqlite3_config() method or by the +# SQLITE_USE_URI compile-time option. # if {$tcl_platform(platform) == "unix"} { set flags [list SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE] @@ -140,8 +141,8 @@ if {$tcl_platform(platform) == "unix"} { } } -# EVIDENCE-OF: R-43804-65312 The 'fragment' component of a URI, if -# present, is always ignored. +# EVIDENCE-OF: R-45981-25528 The fragment component of a URI, if +# present, is ignored. # # It is difficult to test that something is ignore correctly. So these tests # just show that adding a fragment does not interfere with the pathname or @@ -157,14 +158,15 @@ if {$tcl_platform(platform) == "unix"} { } } -# EVIDENCE-OF: R-00273-20588 SQLite uses the 'path' component of the URI -# as the path to the database file to open. +# EVIDENCE-OF: R-62557-09390 SQLite uses the path component of the URI +# as the name of the disk file which contains the database. # # EVIDENCE-OF: R-28659-11035 If the path begins with a '/' character, # then it is interpreted as an absolute path. # -# EVIDENCE-OF: R-39349-47203 If it does not begin with a '/', it is -# interpreted as a relative path. +# EVIDENCE-OF: R-46234-61323 If the path does not begin with a '/' +# (meaning that the authority section is omitted from the URI) then the +# path is interpreted as a relative path. # if {$tcl_platform(platform) == "unix"} { foreach {tn uri parse} " From c3f1d5f0958df10afadc10cd24ea1fb1d7003dfb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 30 May 2011 23:42:16 +0000 Subject: [PATCH 10/69] Minor performance improvements. FossilOrigin-Name: f9950c6af1813f724dacd7455f472acec921b06a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 6 ++++-- src/vdbe.c | 28 ++++++++++++++-------------- src/vdbeaux.c | 4 +++- 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 207bb8d5bf..b9205ae2bb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sevidence\smarks\son\sthe\sURI\sfilename\stests\sto\sconform\sto\sthe\slatest\ndocumentation. -D 2011-05-30T15:06:48.453 +C Minor\sperformance\simprovements. +D 2011-05-30T23:42:16.233 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,7 +122,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 975ad691a57eb1fb60f1ec76ad0b6571eace62f9 +F src/btree.c 0d3b39dcb79565c053e35fc12713f12d8a74d6a9 F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 0132bc6631fa617a1d28ef805921f6dbac18a514 @@ -236,11 +236,11 @@ F src/update.c 5bcb56e5c7380a2eecb0e71891dbd4ad7437748f F src/utf.c d83650c3ea08f7407bd9d0839d9885241c209c60 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 380dddc404c1e4a9260a4194daa728b94ed102d7 +F src/vdbe.c 103827f560cdc48b1d455ce4d4b3573dd88f9ab4 F src/vdbe.h 8a675fefdf7119441fe817c800a9a52440c2e797 F src/vdbeInt.h fe8f58d305e629fff02f61f655aca1d299f1f6ae F src/vdbeapi.c e0e2672e0a96ae3f8575c8ecd02912a3e8a554a1 -F src/vdbeaux.c 07a5226ae6e9c6e54b5fcd3c395b86e7ffdba3a4 +F src/vdbeaux.c 99900868d18618a07ffaf780ecc41fd807834bde F src/vdbeblob.c c3ccb7c8732858c680f442932e66ad06bb036562 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 @@ -938,7 +938,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P e3350dbd9f472c27ea6bc872d85098ac89d874c7 -R 31eb4302622f868d251135572f646bae +P 1bab03c4811b5e5b3d15632bc2a3844891f9fad7 +R 7acb04adb38e3c88e3b1f83ed6f40a4e U drh -Z 47e43c01cd8d7b7136f7bc52a005ac53 +Z 9e474b54141fcb6c3097a32384090f3c diff --git a/manifest.uuid b/manifest.uuid index 45419984ac..58a05fb539 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bab03c4811b5e5b3d15632bc2a3844891f9fad7 \ No newline at end of file +f9950c6af1813f724dacd7455f472acec921b06a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index df75053743..972a2f608c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5390,10 +5390,10 @@ static int fillInCell( ** "sz" must be the number of bytes in the cell. */ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ - int i; /* Loop counter */ u32 pc; /* Offset to cell content of cell being deleted */ u8 *data; /* pPage->aData */ u8 *ptr; /* Used to move bytes around within data[] */ + u8 *endPtr; /* End of loop */ int rc; /* The return code */ int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ @@ -5418,9 +5418,11 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ *pRC = rc; return; } - for(i=idx+1; inCell; i++, ptr+=2){ + endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2]; + while( ptrnCell--; put2byte(&data[hdr+3], pPage->nCell); diff --git a/src/vdbe.c b/src/vdbe.c index d28f73d1b1..eea6aba70e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1766,7 +1766,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ pIn3 = &aMem[pOp->p3]; flags1 = pIn1->flags; flags3 = pIn3->flags; - if( (pIn1->flags | pIn3->flags)&MEM_Null ){ + if( (flags1 | flags3)&MEM_Null ){ /* One or both operands are NULL */ if( pOp->p5 & SQLITE_NULLEQ ){ /* If SQLITE_NULLEQ is set (which will only happen if the operator is @@ -1774,7 +1774,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** or not both operands are null. */ assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); - res = (pIn1->flags & pIn3->flags & MEM_Null)==0; + res = (flags1 & flags3 & MEM_Null)==0; }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL, ** then the result is always NULL. @@ -5882,20 +5882,20 @@ case OP_MaxPgcnt: { /* out2-prerelease */ */ case OP_Trace: { char *zTrace; + char *z; - zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); - if( zTrace ){ - if( db->xTrace ){ - char *z = sqlite3VdbeExpandSql(p, zTrace); - db->xTrace(db->pTraceArg, z); - sqlite3DbFree(db, z); - } -#ifdef SQLITE_DEBUG - if( (db->flags & SQLITE_SqlTrace)!=0 ){ - sqlite3DebugPrintf("SQL-trace: %s\n", zTrace); - } -#endif /* SQLITE_DEBUG */ + if( db->xTrace && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 ){ + z = sqlite3VdbeExpandSql(p, zTrace); + db->xTrace(db->pTraceArg, z); + sqlite3DbFree(db, z); } +#ifdef SQLITE_DEBUG + if( (db->flags & SQLITE_SqlTrace)!=0 + && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 + ){ + sqlite3DebugPrintf("SQL-trace: %s\n", zTrace); + } +#endif /* SQLITE_DEBUG */ break; } #endif diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e9764a5096..0322b04c21 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2852,7 +2852,7 @@ UnpackedRecord *sqlite3VdbeRecordUnpack( idx += getVarint32(&aKey[idx], serial_type); pMem->enc = pKeyInfo->enc; pMem->db = pKeyInfo->db; - pMem->flags = 0; + /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ pMem->zMalloc = 0; d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); pMem++; @@ -2867,6 +2867,7 @@ UnpackedRecord *sqlite3VdbeRecordUnpack( ** This routine destroys a UnpackedRecord object. */ void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ +#ifdef SQLITE_DEBUG int i; Mem *pMem; @@ -2880,6 +2881,7 @@ void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ */ if( NEVER(pMem->zMalloc) ) sqlite3VdbeMemRelease(pMem); } +#endif if( p->flags & UNPACKED_NEED_FREE ){ sqlite3DbFree(p->pKeyInfo->db, p); } From a80a141247a233e71d0adc29244096eebf3204f0 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 May 2011 11:56:15 +0000 Subject: [PATCH 11/69] Update the documentation to state that any parameter that is optimized out of a prepared statement becomes an anonymous parameter for which sqlite3_bind_parameter_name() returns NULL. FossilOrigin-Name: 701b8a23e3ea2c94454af5d9bd1e72acb66d0fe2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b9205ae2bb..f7cee8fa4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sperformance\simprovements. -D 2011-05-30T23:42:16.233 +C Update\sthe\sdocumentation\sto\sstate\sthat\sany\sparameter\sthat\sis\soptimized\sout\nof\sa\sprepared\sstatement\sbecomes\san\sanonymous\sparameter\sfor\swhich\nsqlite3_bind_parameter_name()\sreturns\sNULL. +D 2011-05-31T11:56:15.040 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff F src/shell.c decd04236a7ef26be5ef46d4ea963044bfad9a48 -F src/sqlite.h.in 91c63a69eeddbd62182ec00dbfee390016972bdb +F src/sqlite.h.in c095996e68e0082f674f32e68ca909fc24196a2a F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 F src/sqliteInt.h d9ff5f198b5bac7ee0c6e1ea55f76897ba4dda87 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d @@ -938,7 +938,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1bab03c4811b5e5b3d15632bc2a3844891f9fad7 -R 7acb04adb38e3c88e3b1f83ed6f40a4e +P f9950c6af1813f724dacd7455f472acec921b06a +R f6b845f51167252cf7d20999b01d5085 U drh -Z 9e474b54141fcb6c3097a32384090f3c +Z 659dc0a03ae6f4d76444e7107181eaaf diff --git a/manifest.uuid b/manifest.uuid index 58a05fb539..6374f63920 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9950c6af1813f724dacd7455f472acec921b06a \ No newline at end of file +701b8a23e3ea2c94454af5d9bd1e72acb66d0fe2 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 445a247a19..a9d60e8a2d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3068,6 +3068,8 @@ int sqlite3_bind_parameter_count(sqlite3_stmt*); ** is included as part of the name.)^ ** ^Parameters of the form "?" without a following integer have no name ** and are referred to as "nameless" or "anonymous parameters". +** ^Any parameter that is optimized out of the prepared statement by the +** query planner becomes a nameless or anonymous parameter. ** ** ^The first host parameter has an index of 1, not 0. ** From ed9624187d89a96e591353a7bdee53b292e6f849 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 May 2011 16:50:23 +0000 Subject: [PATCH 12/69] Fix a problem in the sqlite3TestErrorName() function (used only for testing) that appears to have originated from a bad merge. FossilOrigin-Name: a0ae314c7f41d0146a9ee1adc576cd977219a378 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test1.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f7cee8fa4c..9323cd6b55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sdocumentation\sto\sstate\sthat\sany\sparameter\sthat\sis\soptimized\sout\nof\sa\sprepared\sstatement\sbecomes\san\sanonymous\sparameter\sfor\swhich\nsqlite3_bind_parameter_name()\sreturns\sNULL. -D 2011-05-31T11:56:15.040 +C Fix\sa\sproblem\sin\sthe\ssqlite3TestErrorName()\sfunction\s(used\sonly\sfor\stesting)\nthat\sappears\sto\shave\soriginated\sfrom\sa\sbad\smerge. +D 2011-05-31T16:50:23.129 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -186,7 +186,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 501c9a200fd998a268be475be5858febc90b725b -F src/test1.c 4a1171af201be90c21d64a872e686b1333d9a2cf +F src/test1.c 9f61b9d23938bc5dd165ec7b95a91c30ce6e66db F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 @@ -938,7 +938,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P f9950c6af1813f724dacd7455f472acec921b06a -R f6b845f51167252cf7d20999b01d5085 +P 701b8a23e3ea2c94454af5d9bd1e72acb66d0fe2 +R 0252c62ef68072dedfcd687577c244b5 U drh -Z 659dc0a03ae6f4d76444e7107181eaaf +Z 32d78009c9e58634b2f842adf352a1be diff --git a/manifest.uuid b/manifest.uuid index 6374f63920..55d88bed1a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -701b8a23e3ea2c94454af5d9bd1e72acb66d0fe2 \ No newline at end of file +a0ae314c7f41d0146a9ee1adc576cd977219a378 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 50575afbbc..7d31c6d02a 100644 --- a/src/test1.c +++ b/src/test1.c @@ -164,7 +164,6 @@ const char *sqlite3TestErrorName(int rc){ zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; default: zName = "SQLITE_Unknown"; break; } return zName; From 124c0b49a19ce24302c6594e60c80c7ba4df6c98 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Jun 2011 18:15:55 +0000 Subject: [PATCH 13/69] Refactor the SQL parameter processing so that parameter names for values that are optimized out of the prepare statement are not forgotten. FossilOrigin-Name: b3aaf715b60b8a338cc6c92dad1ead4a3f7146a3 --- manifest | 30 +++---- manifest.uuid | 2 +- src/build.c | 4 +- src/expr.c | 86 +++++++++--------- src/sqlite.h.in | 2 - src/sqliteInt.h | 5 +- src/tokenize.c | 8 +- src/vdbe.h | 3 +- src/vdbeInt.h | 2 +- src/vdbeapi.c | 34 +------ src/vdbeaux.c | 229 ++++++++++++++++++++++++++---------------------- src/vdbeblob.c | 5 +- 12 files changed, 202 insertions(+), 208 deletions(-) diff --git a/manifest b/manifest index 9323cd6b55..b747a1409f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\ssqlite3TestErrorName()\sfunction\s(used\sonly\sfor\stesting)\nthat\sappears\sto\shave\soriginated\sfrom\sa\sbad\smerge. -D 2011-05-31T16:50:23.129 +C Refactor\sthe\sSQL\sparameter\sprocessing\sso\sthat\sparameter\snames\sfor\svalues\nthat\sare\soptimized\sout\sof\sthe\sprepare\sstatement\sare\snot\sforgotten. +D 2011-06-01T18:15:55.858 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,13 +125,13 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 0d3b39dcb79565c053e35fc12713f12d8a74d6a9 F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c 0132bc6631fa617a1d28ef805921f6dbac18a514 +F src/build.c c10ab9e2c77ade99dee23554787f8acfc0c231fc F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 F src/date.c 1548fdac51377e4e7833251de878b4058c148e1b F src/delete.c cecc926c70783452f3e8eb452c728291ce1a0b21 -F src/expr.c e3cf0957c6b8faaaf7386a3bc69e53c0dc9705be +F src/expr.c c33584bcf4ee011e28ff74e4dcef02da1dc09dd6 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c a43ba8a005fb5efd1deeee06853e3a6120d46a91 F src/func.c b9117e40975245b8504cf3625d7e321d8d4b63dc @@ -179,9 +179,9 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff F src/shell.c decd04236a7ef26be5ef46d4ea963044bfad9a48 -F src/sqlite.h.in c095996e68e0082f674f32e68ca909fc24196a2a +F src/sqlite.h.in 91c63a69eeddbd62182ec00dbfee390016972bdb F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h d9ff5f198b5bac7ee0c6e1ea55f76897ba4dda87 +F src/sqliteInt.h 6e58c558c57c8f44011736d5fa5295eb3130f9de F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -230,18 +230,18 @@ F src/test_vfs.c e7855568dfa1e0ba73668d273b65605d9f8b77e8 F src/test_vfstrace.c 0b884e06094a746da729119a2cabdc7aa790063d F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/tokenize.c 604607d6813e9551cf5189d899e0a25c12681080 +F src/tokenize.c 7b0c9281b2368dc69135e7a50bd271de9af1b467 F src/trigger.c 144cc18bb701f3286484aae4292a9531f09278c8 F src/update.c 5bcb56e5c7380a2eecb0e71891dbd4ad7437748f F src/utf.c d83650c3ea08f7407bd9d0839d9885241c209c60 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e F src/vdbe.c 103827f560cdc48b1d455ce4d4b3573dd88f9ab4 -F src/vdbe.h 8a675fefdf7119441fe817c800a9a52440c2e797 -F src/vdbeInt.h fe8f58d305e629fff02f61f655aca1d299f1f6ae -F src/vdbeapi.c e0e2672e0a96ae3f8575c8ecd02912a3e8a554a1 -F src/vdbeaux.c 99900868d18618a07ffaf780ecc41fd807834bde -F src/vdbeblob.c c3ccb7c8732858c680f442932e66ad06bb036562 +F src/vdbe.h d9c6123384189dc37d27beac1bf44688aa75b6cb +F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 +F src/vdbeapi.c 0eeadc75e44a30efd996d6af6e7c5a2488e35be8 +F src/vdbeaux.c 0505dc4f7ff3cf35e219fe0a20ab798a42772b8b +F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 9ba8c7fdb7d39260c033a402f6032d3e7bc5d336 @@ -938,7 +938,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 701b8a23e3ea2c94454af5d9bd1e72acb66d0fe2 -R 0252c62ef68072dedfcd687577c244b5 +P a0ae314c7f41d0146a9ee1adc576cd977219a378 +R d99e1065a540709b7ea760de89dbcfd0 U drh -Z 32d78009c9e58634b2f842adf352a1be +Z ce5ce2791f47aa05a4daaafded206f56 diff --git a/manifest.uuid b/manifest.uuid index 55d88bed1a..4597483a7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0ae314c7f41d0146a9ee1adc576cd977219a378 \ No newline at end of file +b3aaf715b60b8a338cc6c92dad1ead4a3f7146a3 \ No newline at end of file diff --git a/src/build.c b/src/build.c index fa2d53a14b..afe2089325 100644 --- a/src/build.c +++ b/src/build.c @@ -200,9 +200,7 @@ void sqlite3FinishCoding(Parse *pParse){ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; - sqlite3VdbeMakeReady(v, pParse->nVar, pParse->nMem, - pParse->nTab, pParse->nMaxArg, pParse->explain, - pParse->isMultiWrite && pParse->mayAbort); + sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; pParse->colNamesSet = 0; }else{ diff --git a/src/expr.c b/src/expr.c index c0e9ba6fd5..4cd2f24551 100644 --- a/src/expr.c +++ b/src/expr.c @@ -555,53 +555,53 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ /* Wildcard of the form "?". Assign the next variable number */ assert( z[0]=='?' ); pExpr->iColumn = (ynVar)(++pParse->nVar); - }else if( z[0]=='?' ){ - /* Wildcard of the form "?nnn". Convert "nnn" to an integer and - ** use it as the variable number */ - i64 i; - int bOk = 0==sqlite3Atoi64(&z[1], &i, sqlite3Strlen30(&z[1]), SQLITE_UTF8); - pExpr->iColumn = (ynVar)i; - testcase( i==0 ); - testcase( i==1 ); - testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); - testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); - if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ - sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", - db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); - } - if( i>pParse->nVar ){ - pParse->nVar = (int)i; - } }else{ - /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable - ** number as the prior appearance of the same name, or if the name - ** has never appeared before, reuse the same variable number - */ - int i; - u32 n; - n = sqlite3Strlen30(z); - for(i=0; inVarExpr; i++){ - Expr *pE = pParse->apVarExpr[i]; - assert( pE!=0 ); - if( memcmp(pE->u.zToken, z, n)==0 && pE->u.zToken[n]==0 ){ - pExpr->iColumn = pE->iColumn; - break; + ynVar x = 0; + u32 n = sqlite3Strlen30(z); + if( z[0]=='?' ){ + /* Wildcard of the form "?nnn". Convert "nnn" to an integer and + ** use it as the variable number */ + i64 i; + int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + pExpr->iColumn = x = (ynVar)i; + testcase( i==0 ); + testcase( i==1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", + db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); + x = 0; } + if( i>pParse->nVar ){ + pParse->nVar = (int)i; + } + }else{ + /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable + ** number as the prior appearance of the same name, or if the name + ** has never appeared before, reuse the same variable number + */ + ynVar i; + for(i=0; inzVar; i++){ + if( pParse->azVar[i] && memcmp(pParse->azVar[i],z,n+1)==0 ){ + pExpr->iColumn = x = (ynVar)i+1; + break; + } + } + if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); } - if( i>=pParse->nVarExpr ){ - pExpr->iColumn = (ynVar)(++pParse->nVar); - if( pParse->nVarExpr>=pParse->nVarExprAlloc-1 ){ - pParse->nVarExprAlloc += pParse->nVarExprAlloc + 10; - pParse->apVarExpr = - sqlite3DbReallocOrFree( - db, - pParse->apVarExpr, - pParse->nVarExprAlloc*sizeof(pParse->apVarExpr[0]) - ); + if( x>0 ){ + if( x>pParse->nzVar ){ + char **a; + a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); + if( a==0 ) return; /* Error reported through db->mallocFailed */ + pParse->azVar = a; + memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); + pParse->nzVar = x; } - if( !db->mallocFailed ){ - assert( pParse->apVarExpr!=0 ); - pParse->apVarExpr[pParse->nVarExpr++] = pExpr; + if( z[0]!='?' || pParse->azVar[x-1]==0 ){ + sqlite3DbFree(db, pParse->azVar[x-1]); + pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); } } } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a9d60e8a2d..445a247a19 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3068,8 +3068,6 @@ int sqlite3_bind_parameter_count(sqlite3_stmt*); ** is included as part of the name.)^ ** ^Parameters of the form "?" without a following integer have no name ** and are referred to as "nameless" or "anonymous parameters". -** ^Any parameter that is optimized out of the prepared statement by the -** query planner becomes a nameless or anonymous parameter. ** ** ^The first host parameter has an index of 1, not 0. ** diff --git a/src/sqliteInt.h b/src/sqliteInt.h index bba0b3c441..fd86158e00 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2230,9 +2230,8 @@ struct Parse { ** each recursion */ int nVar; /* Number of '?' variables seen in the SQL so far */ - int nVarExpr; /* Number of used slots in apVarExpr[] */ - int nVarExprAlloc; /* Number of allocated slots in apVarExpr[] */ - Expr **apVarExpr; /* Pointers to :aaa and $aaaa wildcard expressions */ + int nzVar; /* Number of available slots in azVar[] */ + char **azVar; /* Pointers to names of parameters */ Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ int nAlias; /* Number of aliased result set columns */ int nAliasAlloc; /* Number of allocated slots for aAlias[] */ diff --git a/src/tokenize.c b/src/tokenize.c index c624efdcc7..016a218262 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -412,9 +412,8 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); - assert( pParse->nVarExpr==0 ); - assert( pParse->nVarExprAlloc==0 ); - assert( pParse->apVarExpr==0 ); + assert( pParse->nzVar==0 ); + assert( pParse->azVar==0 ); enableLookaside = db->lookaside.bEnabled; if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; while( !db->mallocFailed && zSql[i]!=0 ){ @@ -508,7 +507,8 @@ abort_parse: } sqlite3DeleteTrigger(db, pParse->pNewTrigger); - sqlite3DbFree(db, pParse->apVarExpr); + for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); + sqlite3DbFree(db, pParse->azVar); sqlite3DbFree(db, pParse->aAlias); while( pParse->pAinc ){ AutoincInfo *p = pParse->pAinc; diff --git a/src/vdbe.h b/src/vdbe.h index 43044533f3..56f9eb51ca 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -185,7 +185,7 @@ int sqlite3VdbeMakeLabel(Vdbe*); void sqlite3VdbeRunOnlyOnce(Vdbe*); void sqlite3VdbeDelete(Vdbe*); void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); -void sqlite3VdbeMakeReady(Vdbe*,int,int,int,int,int,int); +void sqlite3VdbeMakeReady(Vdbe*,Parse*); int sqlite3VdbeFinalize(Vdbe*); void sqlite3VdbeResolveLabel(Vdbe*, int); int sqlite3VdbeCurrentAddr(Vdbe*); @@ -194,6 +194,7 @@ int sqlite3VdbeCurrentAddr(Vdbe*); void sqlite3VdbeTrace(Vdbe*,FILE*); #endif void sqlite3VdbeResetStepResult(Vdbe*); +void sqlite3VdbeRewind(Vdbe*); int sqlite3VdbeReset(Vdbe*); void sqlite3VdbeSetNumCols(Vdbe*,int); int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 5e96c6f9ff..0aeb3af7a9 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -287,11 +287,11 @@ struct Vdbe { Mem *aVar; /* Values for the OP_Variable opcode. */ char **azVar; /* Name of variables */ ynVar nVar; /* Number of entries in aVar[] */ + ynVar nzVar; /* Number of entries in azVar[] */ u32 cacheCtr; /* VdbeCursor row cache generation counter */ int pc; /* The program counter */ int rc; /* Value to return */ u8 errorAction; /* Recovery action to do in case of an error */ - u8 okVar; /* True if azVar[] has been initialized */ u8 explain; /* True if EXPLAIN present on SQL command */ u8 changeCntOn; /* True to update the change-counter */ u8 expired; /* True if the VM needs to be recompiled */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 90baaccc68..5923a4c01f 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -102,7 +102,7 @@ int sqlite3_reset(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; sqlite3_mutex_enter(v->db->mutex); rc = sqlite3VdbeReset(v); - sqlite3VdbeMakeReady(v, -1, 0, 0, 0, 0, 0); + sqlite3VdbeRewind(v); assert( (rc & (v->db->errMask))==rc ); rc = sqlite3ApiExit(v->db, rc); sqlite3_mutex_leave(v->db->mutex); @@ -1167,32 +1167,6 @@ int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ return p ? p->nVar : 0; } -/* -** Create a mapping from variable numbers to variable names -** in the Vdbe.azVar[] array, if such a mapping does not already -** exist. -*/ -static void createVarMap(Vdbe *p){ - if( !p->okVar ){ - int j; - Op *pOp; - sqlite3_mutex_enter(p->db->mutex); - /* The race condition here is harmless. If two threads call this - ** routine on the same Vdbe at the same time, they both might end - ** up initializing the Vdbe.azVar[] array. That is a little extra - ** work but it results in the same answer. - */ - for(j=0, pOp=p->aOp; jnOp; j++, pOp++){ - if( pOp->opcode==OP_Variable ){ - assert( pOp->p1>0 && pOp->p1<=p->nVar ); - p->azVar[pOp->p1-1] = pOp->p4.z; - } - } - p->okVar = 1; - sqlite3_mutex_leave(p->db->mutex); - } -} - /* ** Return the name of a wildcard parameter. Return NULL if the index ** is out of range or if the wildcard is unnamed. @@ -1201,10 +1175,9 @@ static void createVarMap(Vdbe *p){ */ const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ Vdbe *p = (Vdbe*)pStmt; - if( p==0 || i<1 || i>p->nVar ){ + if( p==0 || i<1 || i>p->nzVar ){ return 0; } - createVarMap(p); return p->azVar[i-1]; } @@ -1218,9 +1191,8 @@ int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ if( p==0 ){ return 0; } - createVarMap(p); if( zName ){ - for(i=0; inVar; i++){ + for(i=0; inzVar; i++){ const char *z = p->azVar[i]; if( z && memcmp(z,zName,nName)==0 && z[nName]==0 ){ return i+1; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0322b04c21..8181f01aa3 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1391,34 +1391,13 @@ static void *allocSpace( } /* -** Prepare a virtual machine for execution. This involves things such -** as allocating stack space and initializing the program counter. -** After the VDBE has be prepped, it can be executed by one or more -** calls to sqlite3VdbeExec(). -** -** This is the only way to move a VDBE from VDBE_MAGIC_INIT to -** VDBE_MAGIC_RUN. -** -** This function may be called more than once on a single virtual machine. -** The first call is made while compiling the SQL statement. Subsequent -** calls are made as part of the process of resetting a statement to be -** re-executed (from a call to sqlite3_reset()). The nVar, nMem, nCursor -** and isExplain parameters are only passed correct values the first time -** the function is called. On subsequent calls, from sqlite3_reset(), nVar -** is passed -1 and nMem, nCursor and isExplain are all passed zero. +** Rewind the VDBE back to the beginning in preparation for +** running it. */ -void sqlite3VdbeMakeReady( - Vdbe *p, /* The VDBE */ - int nVar, /* Number of '?' see in the SQL statement */ - int nMem, /* Number of memory cells to allocate */ - int nCursor, /* Number of cursors to allocate */ - int nArg, /* Maximum number of args in SubPrograms */ - int isExplain, /* True if the EXPLAIN keywords is present */ - int usesStmtJournal /* True to set Vdbe.usesStmtJournal */ -){ - int n; - sqlite3 *db = p->db; - +void sqlite3VdbeRewind(Vdbe *p){ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + int i; +#endif assert( p!=0 ); assert( p->magic==VDBE_MAGIC_INIT ); @@ -1429,6 +1408,71 @@ void sqlite3VdbeMakeReady( /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. */ p->magic = VDBE_MAGIC_RUN; +#ifdef SQLITE_DEBUG + for(i=1; inMem; i++){ + assert( p->aMem[i].db==p->db ); + } +#endif + p->pc = -1; + p->rc = SQLITE_OK; + p->errorAction = OE_Abort; + p->magic = VDBE_MAGIC_RUN; + p->nChange = 0; + p->cacheCtr = 1; + p->minWriteFileFormat = 255; + p->iStatement = 0; + p->nFkConstraint = 0; +#ifdef VDBE_PROFILE + for(i=0; inOp; i++){ + p->aOp[i].cnt = 0; + p->aOp[i].cycles = 0; + } +#endif +} + +/* +** Prepare a virtual machine for execution for the first time after +** creating the virtual machine. This involves things such +** as allocating stack space and initializing the program counter. +** After the VDBE has be prepped, it can be executed by one or more +** calls to sqlite3VdbeExec(). +** +** This function may be called exact once on a each virtual machine. +** After this routine is called the VM has been "packaged" and is ready +** to run. After this routine is called, futher calls to +** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects +** the Vdbe from the Parse object that helped generate it so that the +** the Vdbe becomes an independent entity and the Parse object can be +** destroyed. +** +** Use the sqlite3VdbeRewind() procedure to restore a virtual machine back +** to its initial state after it has been run. +*/ +void sqlite3VdbeMakeReady( + Vdbe *p, /* The VDBE */ + Parse *pParse /* Parsing context */ +){ + sqlite3 *db; /* The database connection */ + int nVar; /* Number of parameters */ + int nMem; /* Number of VM memory registers */ + int nCursor; /* Number of cursors required */ + int nArg; /* Number of arguments in subprograms */ + int n; /* Loop counter */ + u8 *zCsr; /* Memory available for allocation */ + u8 *zEnd; /* First byte past allocated memory */ + int nByte; /* How much extra memory is needed */ + + assert( p!=0 ); + assert( p->nOp>0 ); + assert( pParse!=0 ); + assert( p->magic==VDBE_MAGIC_INIT ); + db = p->db; + assert( db->mallocFailed==0 ); + nVar = pParse->nVar; + nMem = pParse->nMem; + nCursor = pParse->nTab; + nArg = pParse->nMaxArg; + /* For each cursor required, also allocate a memory cell. Memory ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by ** the vdbe program. Instead they are used to allocate space for @@ -1441,91 +1485,68 @@ void sqlite3VdbeMakeReady( nMem += nCursor; /* Allocate space for memory registers, SQL variables, VDBE cursors and - ** an array to marshal SQL function arguments in. This is only done the - ** first time this function is called for a given VDBE, not when it is - ** being called from sqlite3_reset() to reset the virtual machine. + ** an array to marshal SQL function arguments in. */ - if( nVar>=0 && ALWAYS(db->mallocFailed==0) ){ - u8 *zCsr = (u8 *)&p->aOp[p->nOp]; /* Memory avaliable for alloation */ - u8 *zEnd = (u8 *)&p->aOp[p->nOpAlloc]; /* First byte past available mem */ - int nByte; /* How much extra memory needed */ + zCsr = (u8*)&p->aOp[p->nOp]; /* Memory avaliable for allocation */ + zEnd = (u8*)&p->aOp[p->nOpAlloc]; /* First byte past end of zCsr[] */ - resolveP2Values(p, &nArg); - p->usesStmtJournal = (u8)usesStmtJournal; - if( isExplain && nMem<10 ){ - nMem = 10; + resolveP2Values(p, &nArg); + p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); + if( pParse->explain && nMem<10 ){ + nMem = 10; + } + memset(zCsr, 0, zEnd-zCsr); + zCsr += (zCsr - (u8*)0)&7; + assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + + /* Memory for registers, parameters, cursor, etc, is allocated in two + ** passes. On the first pass, we try to reuse unused space at the + ** end of the opcode array. If we are unable to satisfy all memory + ** requirements by reusing the opcode array tail, then the second + ** pass will fill in the rest using a fresh allocation. + ** + ** This two-pass approach that reuses as much memory as possible from + ** the leftover space at the end of the opcode array can significantly + ** reduce the amount of memory held by a prepared statement. + */ + do { + nByte = 0; + p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); + p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); + p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); + p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); + p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), + &zCsr, zEnd, &nByte); + if( nByte ){ + p->pFree = sqlite3DbMallocZero(db, nByte); } - memset(zCsr, 0, zEnd-zCsr); - zCsr += (zCsr - (u8*)0)&7; - assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + zCsr = p->pFree; + zEnd = &zCsr[nByte]; + }while( nByte && !db->mallocFailed ); - /* Memory for registers, parameters, cursor, etc, is allocated in two - ** passes. On the first pass, we try to reuse unused space at the - ** end of the opcode array. If we are unable to satisfy all memory - ** requirements by reusing the opcode array tail, then the second - ** pass will fill in the rest using a fresh allocation. - ** - ** This two-pass approach that reuses as much memory as possible from - ** the leftover space at the end of the opcode array can significantly - ** reduce the amount of memory held by a prepared statement. - */ - do { - nByte = 0; - p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); - p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); - p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); - p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); - p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), - &zCsr, zEnd, &nByte); - if( nByte ){ - p->pFree = sqlite3DbMallocZero(db, nByte); - } - zCsr = p->pFree; - zEnd = &zCsr[nByte]; - }while( nByte && !db->mallocFailed ); - - p->nCursor = (u16)nCursor; - if( p->aVar ){ - p->nVar = (ynVar)nVar; - for(n=0; naVar[n].flags = MEM_Null; - p->aVar[n].db = db; - } - } - if( p->aMem ){ - p->aMem--; /* aMem[] goes from 1..nMem */ - p->nMem = nMem; /* not from 0..nMem-1 */ - for(n=1; n<=nMem; n++){ - p->aMem[n].flags = MEM_Null; - p->aMem[n].db = db; - } + p->nCursor = (u16)nCursor; + if( p->aVar ){ + p->nVar = (ynVar)nVar; + for(n=0; naVar[n].flags = MEM_Null; + p->aVar[n].db = db; } } -#ifdef SQLITE_DEBUG - for(n=1; nnMem; n++){ - assert( p->aMem[n].db==db ); + if( p->azVar ){ + p->nzVar = pParse->nzVar; + memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); + memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); } -#endif - - p->pc = -1; - p->rc = SQLITE_OK; - p->errorAction = OE_Abort; - p->explain |= isExplain; - p->magic = VDBE_MAGIC_RUN; - p->nChange = 0; - p->cacheCtr = 1; - p->minWriteFileFormat = 255; - p->iStatement = 0; - p->nFkConstraint = 0; -#ifdef VDBE_PROFILE - { - int i; - for(i=0; inOp; i++){ - p->aOp[i].cnt = 0; - p->aOp[i].cycles = 0; + if( p->aMem ){ + p->aMem--; /* aMem[] goes from 1..nMem */ + p->nMem = nMem; /* not from 0..nMem-1 */ + for(n=1; n<=nMem; n++){ + p->aMem[n].flags = MEM_Null; + p->aMem[n].db = db; } } -#endif + p->explain = pParse->explain; + sqlite3VdbeRewind(p); } /* @@ -2399,6 +2420,7 @@ void sqlite3VdbeDeleteAuxData(VdbeFunc *pVdbeFunc, int mask){ */ void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; + int i; assert( p->db==0 || p->db==db ); releaseMemArray(p->aVar, p->nVar); releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); @@ -2407,6 +2429,7 @@ void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); sqlite3DbFree(db, pSub); } + for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aLabel); sqlite3DbFree(db, p->aColName); diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 18fdd465ae..a8728e6d25 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -297,7 +297,10 @@ int sqlite3_blob_open( sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); sqlite3VdbeChangeP2(v, 7, pTab->nCol); if( !db->mallocFailed ){ - sqlite3VdbeMakeReady(v, 1, 1, 1, 0, 0, 0); + pParse->nVar = 1; + pParse->nMem = 1; + pParse->nTab = 1; + sqlite3VdbeMakeReady(v, pParse); } } From 04e9eeadc63bb651a658150e0529cd3463914690 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Jun 2011 19:16:06 +0000 Subject: [PATCH 14/69] Avoid unnecessary duplication of SQL parameter names. FossilOrigin-Name: e704e8690ae35decc9769a45cf8d519ccad8b79d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 4 +++- src/vdbe.c | 1 + 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b747a1409f..9e6489e647 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\sSQL\sparameter\sprocessing\sso\sthat\sparameter\snames\sfor\svalues\nthat\sare\soptimized\sout\sof\sthe\sprepare\sstatement\sare\snot\sforgotten. -D 2011-06-01T18:15:55.858 +C Avoid\sunnecessary\sduplication\sof\sSQL\sparameter\snames. +D 2011-06-01T19:16:06.364 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -131,7 +131,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 F src/date.c 1548fdac51377e4e7833251de878b4058c148e1b F src/delete.c cecc926c70783452f3e8eb452c728291ce1a0b21 -F src/expr.c c33584bcf4ee011e28ff74e4dcef02da1dc09dd6 +F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c a43ba8a005fb5efd1deeee06853e3a6120d46a91 F src/func.c b9117e40975245b8504cf3625d7e321d8d4b63dc @@ -236,7 +236,7 @@ F src/update.c 5bcb56e5c7380a2eecb0e71891dbd4ad7437748f F src/utf.c d83650c3ea08f7407bd9d0839d9885241c209c60 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 103827f560cdc48b1d455ce4d4b3573dd88f9ab4 +F src/vdbe.c 2ec37637fa70ab0c694d8327ee5dcedbc0621524 F src/vdbe.h d9c6123384189dc37d27beac1bf44688aa75b6cb F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 F src/vdbeapi.c 0eeadc75e44a30efd996d6af6e7c5a2488e35be8 @@ -938,7 +938,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a0ae314c7f41d0146a9ee1adc576cd977219a378 -R d99e1065a540709b7ea760de89dbcfd0 +P b3aaf715b60b8a338cc6c92dad1ead4a3f7146a3 +R 018490a374b5cf414074a5a4bbd3de70 U drh -Z ce5ce2791f47aa05a4daaafded206f56 +Z dcf10ad87997753baf45956f9011da61 diff --git a/manifest.uuid b/manifest.uuid index 4597483a7e..de373954a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b3aaf715b60b8a338cc6c92dad1ead4a3f7146a3 \ No newline at end of file +e704e8690ae35decc9769a45cf8d519ccad8b79d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 4cd2f24551..be2f4d7c66 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2345,7 +2345,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); if( pExpr->u.zToken[1]!=0 ){ - sqlite3VdbeChangeP4(v, -1, pExpr->u.zToken, P4_TRANSIENT); + assert( pExpr->u.zToken[0]=='?' + || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); + sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); } break; } diff --git a/src/vdbe.c b/src/vdbe.c index eea6aba70e..439fe72540 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -988,6 +988,7 @@ case OP_Variable: { /* out2-prerelease */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); + assert( pOp->p4.z==0 || pOp->p4.z==p->azVar[pOp->p1-1] ); pVar = &p->aVar[pOp->p1 - 1]; if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; From 66dfec8b7a6052d4b639e5ab1910249aebb8828d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Jun 2011 20:01:49 +0000 Subject: [PATCH 15/69] Simplify the wal-readonly branch so that it does not require changes to anything other than os_unix.c and wal.c and a couple of new error codes. FossilOrigin-Name: d6b4709de4d1f8af001f58938247f00a652a616e --- manifest | 30 ++++++++++----------- manifest.uuid | 2 +- src/attach.c | 6 ++--- src/btree.h | 1 - src/main.c | 18 ++----------- src/os_unix.c | 69 ++++++++++++++++++++----------------------------- src/pager.c | 7 ++--- src/pager.h | 1 - src/sqlite.h.in | 41 +---------------------------- src/sqliteInt.h | 2 +- src/wal.c | 62 ++++++++++++++------------------------------ src/wal.h | 4 +-- 12 files changed, 74 insertions(+), 169 deletions(-) diff --git a/manifest b/manifest index b3a84d3a3f..8b6e11236e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Pull\sthe\slatest\strunk\schanges\sinto\sthe\swal-readonly\sbranch. -D 2011-06-01T19:44:57.308 +C Simplify\sthe\swal-readonly\sbranch\sso\sthat\sit\sdoes\snot\srequire\schanges\sto\nanything\sother\sthan\sos_unix.c\sand\swal.c\sand\sa\scouple\sof\snew\serror\scodes. +D 2011-06-01T20:01:49.650 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -117,13 +117,13 @@ F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 280f5c04b11b492703a342222b3de0a999445280 F src/analyze.c a425d62e8fa9ebcb4359ab84ff0c62c6563d2e2a -F src/attach.c a87bfb77a720f7aa02791434aacfd9bc8feb50cc +F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 0d3b39dcb79565c053e35fc12713f12d8a74d6a9 -F src/btree.h d796dc4030b3cce7f5a0cc4f2f986d2befa6b8ac +F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c c10ab9e2c77ade99dee23554787f8acfc0c231fc F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a @@ -144,7 +144,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 3ae0d52da013a6326310655be6473fd472347b85 -F src/main.c 4e55ff4800181e3ad063492740f54ce406de5deb +F src/main.c 059daeed5876b3604f0192f838faf5f4db138901 F src/malloc.c 591aedb20ae40813f1045f2ef253438a334775d9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 @@ -163,10 +163,10 @@ F src/os.c 22ac61d06e72a0dac900400147333b07b13d8e1d F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c af3c3f6a0fc6dbde802e55848f1cad7fb2121ff3 +F src/os_unix.c a59c0718021934157f235468788c793cbc0d53de F src/os_win.c 218b899469e570d46eb8147c2383075f7c026230 -F src/pager.c 2cf3bad86e6fbac42d9dbd3b89799185a343b17b -F src/pager.h 34c6b029446f06f40847746d22faac0d354dd909 +F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 +F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 @@ -179,9 +179,9 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff F src/shell.c decd04236a7ef26be5ef46d4ea963044bfad9a48 -F src/sqlite.h.in dbe88418628302c0a93671a216346acb5746211a +F src/sqlite.h.in 2f51e4f58b2b4626fcbd9938580e730cb5fb4985 F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h 219b21e80f46eef7cc7bb7477c12de61d00f5607 +F src/sqliteInt.h 6e58c558c57c8f44011736d5fa5295eb3130f9de F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -245,8 +245,8 @@ F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 9ba8c7fdb7d39260c033a402f6032d3e7bc5d336 -F src/wal.c 861ea98779dac3867e5f97d13e61cbbb39b310b0 -F src/wal.h 0835021ae243c2f1119e997e8af5d8cc3b991de1 +F src/wal.c fd63d07233203dd3bd29cbe1ae5c8bb2c34e08fc +F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 2c6b5a28e3f6b7cb96b944d0a254f3707885f1ce e704e8690ae35decc9769a45cf8d519ccad8b79d -R 5f4edf2ff77d011b562e3dc234f8f21d +P 0b63b71357a65e26ecd3f3bb34a5f14feee322f4 +R 20a045d8e6470f5f17b67b2f15ff8859 U drh -Z 99212444a62258034d706d358ee0a50e +Z f7b3b70c555e3fb5f3360ed8e60c2877 diff --git a/manifest.uuid b/manifest.uuid index 70ad59250f..6f393899aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b63b71357a65e26ecd3f3bb34a5f14feee322f4 \ No newline at end of file +d6b4709de4d1f8af001f58938247f00a652a616e \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index 4b8ea77917..18f8823b31 100644 --- a/src/attach.c +++ b/src/attach.c @@ -76,8 +76,6 @@ static void attachFunc( Db *aNew; char *zErrDyn = 0; sqlite3_vfs *pVfs; - const char *zVfs = db->pVfs->zName; /* Name of default (main) VFS */ - int btflags = 0; UNUSED_PARAMETER(NotUsed); @@ -131,7 +129,7 @@ static void attachFunc( ** or may not be initialised. */ flags = db->openFlags; - rc = sqlite3ParseUri(zVfs, zFile, &flags, &btflags, &pVfs, &zPath, &zErr); + rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; sqlite3_result_error(context, zErr, -1); @@ -140,7 +138,7 @@ static void attachFunc( } assert( pVfs ); flags |= SQLITE_OPEN_MAIN_DB; - rc = sqlite3BtreeOpen(pVfs, zPath, db, &aNew->pBt, btflags, flags); + rc = sqlite3BtreeOpen(pVfs, zPath, db, &aNew->pBt, 0, flags); sqlite3_free( zPath ); db->nDb++; if( rc==SQLITE_CONSTRAINT ){ diff --git a/src/btree.h b/src/btree.h index c5abc9c865..9e3a73b3b6 100644 --- a/src/btree.h +++ b/src/btree.h @@ -61,7 +61,6 @@ int sqlite3BtreeOpen( #define BTREE_MEMORY 4 /* This is an in-memory DB */ #define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ #define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ -#define BTREE_READONLYSHM 32 /* Read-only SHM access is acceptable */ int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); diff --git a/src/main.c b/src/main.c index cdd54b659d..c99d396546 100644 --- a/src/main.c +++ b/src/main.c @@ -1803,11 +1803,6 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ ** *pFlags may be updated before returning if the URI filename contains ** "cache=xxx" or "mode=xxx" query parameters. ** -** The third argument, pBtflags, points to an integer containing the flags -** that will be passed as the 5th argument to sqlite3BtreeOpen (BTREE_XXX -** flags). This value will be edited if the URI filename contains a -** "readonly_shm=1" or "readonly_shm=0" query parameter. -** ** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to ** the VFS that should be used to open the database file. *pzFile is set to ** point to a buffer containing the name of the file to open. It is the @@ -1823,7 +1818,6 @@ int sqlite3ParseUri( const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ const char *zUri, /* Nul-terminated URI to parse */ unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ - int *pBtflags, /* IN/OUT: BTREE_XXX flags */ sqlite3_vfs **ppVfs, /* OUT: VFS to use */ char **pzFile, /* OUT: Filename component of URI */ char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ @@ -1939,12 +1933,6 @@ int sqlite3ParseUri( if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ zVfs = zVal; - }else if( nOpt==12 && memcmp("readonly_shm", zOpt, 12)==0 ){ - if( sqlite3Atoi(zVal) ){ - *pBtflags |= BTREE_READONLYSHM; - }else{ - *pBtflags &= ~BTREE_READONLYSHM; - } }else{ struct OpenMode { const char *z; @@ -2048,7 +2036,6 @@ static int openDatabase( int isThreadsafe; /* True for threadsafe connections */ char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ - int btflags = 0; /* Mask of BTREE_XXX flags */ *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT @@ -2177,8 +2164,7 @@ static int openDatabase( /* Parse the filename/URI argument. */ db->openFlags = flags; - rc = sqlite3ParseUri( - zVfs, zFilename, &flags, &btflags, &db->pVfs, &zOpen, &zErrMsg); + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; sqlite3Error(db, rc, zErrMsg ? "%s" : 0, zErrMsg); @@ -2187,7 +2173,7 @@ static int openDatabase( } /* Open the backend database driver */ - rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, btflags, + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, flags | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ if( rc==SQLITE_IOERR_NOMEM ){ diff --git a/src/os_unix.c b/src/os_unix.c index 947f431aeb..e6d1774a49 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -212,7 +212,6 @@ struct unixFile { UnixUnusedFd *pUnused; /* Pre-allocated UnixUnusedFd */ const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ - int readOnlyShm; /* True to open shared-memory read-only */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ #if SQLITE_ENABLE_LOCKING_STYLE int openFlags; /* The flags specified at open() */ @@ -3453,10 +3452,6 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ case SQLITE_FCNTL_SIZE_HINT: { return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); } - case SQLITE_FCNTL_READONLY_SHM: { - ((unixFile*)id)->readOnlyShm = (pArg!=0); - return SQLITE_OK; - } #ifndef NDEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and @@ -3542,11 +3537,11 @@ struct unixShmNode { char *zFilename; /* Name of the mmapped file */ int h; /* Open file descriptor */ int szRegion; /* Size of shared-memory regions */ - int nRegion; /* Size of array apRegion */ + u16 nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ - u8 readOnly; /* True if this is a read-only mapping */ #ifdef SQLITE_DEBUG u8 exclMask; /* Mask of exclusive locks held */ u8 sharedMask; /* Mask of shared locks held */ @@ -3787,48 +3782,38 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ - int flags = (pDbFd->readOnlyShm ? O_RDONLY : O_RDWR|O_CREAT); - pShmNode->h = robust_open(zShmFilename, flags, (sStat.st_mode & 0777)); + pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, + (sStat.st_mode & 0777)); if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; + const char *zRO; + zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); + if( zRO && (zRO[0]!='0' || zRO[1]!=0) ){ + pShmNode->h = robust_open(zShmFilename, O_RDONLY, + (sStat.st_mode & 0777)); + pShmNode->isReadonly = 1; + } + if( pShmNode->h<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); + goto shm_open_err; + } } - pShmNode->readOnly = pDbFd->readOnlyShm; /* Check to see if another process is holding the dead-man switch. - ** If not, zero the first few bytes of the shared-memory file to make - ** sure it is not mistaken for valid by code in wal.c. Except, if this - ** is a read-only connection to the shared-memory then it is not possible - ** to check check if another process is holding a read-lock on the DMS - ** byte, as we cannot attempt a write-lock via a read-only file - ** descriptor. So in this case, we just assume the shared-memory - ** contents are Ok and proceed. */ - if( pShmNode->readOnly==0 ){ - rc = SQLITE_OK; - if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ - if( pDbFd->readOnlyShm ){ - rc = SQLITE_IOERR_SHMOPEN; - }else if( 4!=osWrite(pShmNode->h, "\00\00\00\00", 4) ){ - rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); - } + ** If not, truncate the file to zero length. + */ + rc = SQLITE_OK; + if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ + if( robust_ftruncate(pShmNode->h, 0) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); } - if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; } + if( rc==SQLITE_OK ){ + rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1); + } + if( rc ) goto shm_open_err; } } - /* If the unixShmNode is read-only, but SQLITE_FCNTL_READONLY_SHM has not - ** been set for file-descriptor pDbFd, return an error. The wal.c module - ** will then call this function again with SQLITE_FCNTL_READONLY_SHM set. - */ - else if( pShmNode->readOnly && !pDbFd->readOnlyShm ){ - rc = SQLITE_IOERR_SHMOPEN; - goto shm_open_err; - } - /* Make the new connection a child of the unixShmNode */ p->pShmNode = pShmNode; #ifdef SQLITE_DEBUG @@ -3949,7 +3934,8 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, PROT_READ|(!pShmNode->readOnly?PROT_WRITE:0), + pMem = mmap(0, szRegion, + pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion ); if( pMem==MAP_FAILED ){ @@ -3975,6 +3961,7 @@ shmpage_out: }else{ *pp = 0; } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; sqlite3_mutex_leave(pShmNode->mutex); return rc; } diff --git a/src/pager.c b/src/pager.c index 461c735432..7ff9a9a000 100644 --- a/src/pager.c +++ b/src/pager.c @@ -620,7 +620,6 @@ struct Pager { u8 tempFile; /* zFilename is a temporary file */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ - u8 readOnlyShm; /* True if read-only shm access is Ok */ /************************************************************************** ** The following block contains those class members that change during @@ -3018,7 +3017,6 @@ static int pagerWalFrames( static int pagerBeginReadTransaction(Pager *pPager){ int rc; /* Return code */ int changed = 0; /* True if cache must be reset */ - Wal *pWal = pPager->pWal; assert( pagerUseWal(pPager) ); assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); @@ -3028,9 +3026,9 @@ static int pagerBeginReadTransaction(Pager *pPager){ ** are in locking_mode=NORMAL and EndRead() was previously called, ** the duplicate call is harmless. */ - sqlite3WalEndReadTransaction(pWal); + sqlite3WalEndReadTransaction(pPager->pWal); - rc = sqlite3WalBeginReadTransaction(pWal, pPager->readOnlyShm, &changed); + rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); } @@ -4418,7 +4416,6 @@ int sqlite3PagerOpen( } pPager->pVfs = pVfs; pPager->vfsFlags = vfsFlags; - pPager->readOnlyShm = (flags & PAGER_READONLYSHM)!=0; /* Open the pager file. */ diff --git a/src/pager.h b/src/pager.h index c6cb8bcb55..eab7ddaf80 100644 --- a/src/pager.h +++ b/src/pager.h @@ -60,7 +60,6 @@ typedef struct PgHdr DbPage; #define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ #define PAGER_NO_READLOCK 0x0002 /* Omit readlocks on readonly files */ #define PAGER_MEMORY 0x0004 /* In-memory database */ -#define PAGER_READONLYSHM 0x0020 /* Read-only SHM access is acceptable */ /* ** Valid values for the second argument to sqlite3PagerLockingMode(). diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 45aec0e1c2..593e9148fe 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -454,7 +454,6 @@ int sqlite3_exec( #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) - #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) @@ -737,27 +736,6 @@ struct sqlite3_io_methods { ** Applications should not call [sqlite3_file_control()] with this ** opcode as doing so may disrupt the operation of the specialized VFSes ** that do require it. -** -** The [SQLITE_FCNTL_READONLY_SHM] may be generated internally by SQLite if -** the "readonly_shm=1" URI option is specified when the database is opened. -** The fourth argument passed to the VFS xFileControl methods is a pointer -** to a variable of type "int" containing the value 1 or 0. If the variable -** contains the value 1, then this indicates to the VFS that a read-only -** mapping of the shared-memory region is acceptable. If it is set to 0, then -** this indicates that a read-write mapping is required (as normal). If -** a read-only mapping is returned, then the VFS may also return read-only -** mappings for any subsequent requests via the same file-descriptor - -** regardless of the value most recently configured using -** SQLITE_FCNTL_READONLY_SHM. -** -** In practice, if "readonly_shm=1" is specified and the first attempt to -** map a shared-memory region fails, then this file-control is invoked with -** the argument variable set to 1 and a second attempt to map the shared-memory -** region is made. If this mapping succeeds, then the connection continues -** with the read-only mapping. Otherwise, if it fails, SQLITE_CANTOPEN is -** returned to the caller. Whether or not the second (read-only) mapping -** attempt succeeds, the file-control is invoked again with the argument -** variable set to 0. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 @@ -767,7 +745,6 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_CHUNK_SIZE 6 #define SQLITE_FCNTL_FILE_POINTER 7 #define SQLITE_FCNTL_SYNC_OMITTED 8 -#define SQLITE_FCNTL_READONLY_SHM 9 /* @@ -2481,7 +2458,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** [[core URI query parameters]] ** The query component of a URI may contain parameters that are interpreted ** either by SQLite itself, or by a [VFS | custom VFS implementation]. -** SQLite interprets the following four query parameters: +** SQLite interprets the following three query parameters: ** **
    **
  • vfs: ^The "vfs" parameter may be used to specify the name of @@ -2513,22 +2490,6 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in ** a URI filename, its value overrides any behaviour requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. -** -**
  • readonly_shm: ^The readonly_shm parameter may be set to -** either "1" or "0". Setting it to "1" indicates that if the database -** is in WAL mode and read-write access to the associated shared -** memory region cannot be obtained, then an attempt should be made to open -** the shared-memory in read-only mode instead. If there exist one or -** more other database clients that have read-write connections to the -** database shared-memory, then a read-only shared-memory connection works -** fine. However, if there exist no clients with read-write connections -** to the shared-memory and the most recent such crashed or was interrupted -** by a power failure, then it is possible for a database client using a -** read-only connection to return incorrect data, incorrectly report -** database corruption, or return an SQLITE_READONLY error. Or if the -** most recent read-write connection shut down cleanly, it may not be -** possible to open the shared-memory in read-only mode at all, and SQLite -** will return SQLITE_CANTOPEN. **
** ** ^Specifying an unknown parameter in the query component of a URI is not an diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8b0aa855d8..fd86158e00 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2673,7 +2673,7 @@ void sqlite3AddColumnType(Parse*,Token*); void sqlite3AddDefaultValue(Parse*,ExprSpan*); void sqlite3AddCollateType(Parse*, Token*); void sqlite3EndTable(Parse*,Token*,Token*,Select*); -int sqlite3ParseUri(const char*,const char*,unsigned int*,int*, +int sqlite3ParseUri(const char*,const char*,unsigned int*, sqlite3_vfs**,char**,char **); Bitvec *sqlite3BitvecCreate(u32); diff --git a/src/wal.c b/src/wal.c index 87d3fcc968..9f9e1115fa 100644 --- a/src/wal.c +++ b/src/wal.c @@ -406,14 +406,6 @@ struct WalCkptInfo { /* ** An open write-ahead log file is represented by an instance of the ** following object. -** -** The readOnlyShm variable is normally set to 0. If it is set to 1, then -** the connection to shared-memory is read-only. This means it cannot -** be written at all (even when read-locking the database). If it is set -** to 2, then the shared-memory region is not yet open, but a read-only -** connection is acceptable. In this case when the shared-memory is opened -** (see function walIndexPage()), readOnlyShm is set to either 0 or 1 as -** appropriate. */ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ @@ -428,8 +420,7 @@ struct Wal { u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ u8 writeLock; /* True if in a write transaction */ u8 ckptLock; /* True if holding a checkpoint lock */ - u8 readOnly; /* True if the WAL file is open read-only */ - u8 readOnlyShm; /* True if the SHM file is open read-only */ + u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ WalIndexHdr hdr; /* Wal-index header for current transaction */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ @@ -445,6 +436,13 @@ struct Wal { #define WAL_EXCLUSIVE_MODE 1 #define WAL_HEAPMEMORY_MODE 2 +/* +** Possible values for WAL.readOnly +*/ +#define WAL_RDWR 0 /* Normal read/write connection */ +#define WAL_RDONLY 1 /* The WAL file is readonly */ +#define WAL_SHM_RDONLY 2 /* The SHM file is readonly */ + /* ** Each page of the wal-index mapping contains a hash-table made up of ** an array of HASHTABLE_NSLOT elements of the following type. @@ -538,23 +536,15 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); - if( rc==SQLITE_CANTOPEN && pWal->readOnlyShm>1 ){ - assert( iPage==0 ); - sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_READONLY_SHM, (void*)1); - rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, - pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] - ); - if( rc==SQLITE_OK ){ - pWal->readOnly = pWal->readOnlyShm = 1; - } - sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_READONLY_SHM, (void*)0); + if( rc==SQLITE_READONLY ){ + pWal->readOnly |= WAL_SHM_RDONLY; + rc = SQLITE_OK; } } } - *ppPage = pWal->apWiData[iPage]; + *ppPage = pWal->apWiData[iPage]; assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); - if( pWal->readOnlyShm>1 ) pWal->readOnlyShm = 0; return rc; } @@ -794,7 +784,6 @@ static void walUnlockShared(Wal *pWal, int lockIdx){ } static int walLockExclusive(Wal *pWal, int lockIdx, int n){ int rc; - assert( pWal->readOnlyShm==0 ); if( pWal->exclusiveMode ) return SQLITE_OK; rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); @@ -804,7 +793,6 @@ static int walLockExclusive(Wal *pWal, int lockIdx, int n){ return rc; } static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ - assert( pWal->readOnlyShm==0 ); if( pWal->exclusiveMode ) return; (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); @@ -1080,7 +1068,6 @@ static int walIndexRecover(Wal *pWal){ assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 ); assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); assert( pWal->writeLock ); - assert( pWal->readOnlyShm==0 ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; nLock = SQLITE_SHM_NLOCK - iLock; rc = walLockExclusive(pWal, iLock, nLock); @@ -1300,7 +1287,7 @@ int sqlite3WalOpen( flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ - pRet->readOnly = 1; + pRet->readOnly = WAL_RDONLY; } if( rc!=SQLITE_OK ){ @@ -1929,7 +1916,6 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ return rc; }; assert( page0 || pWal->writeLock==0 ); - assert( pWal->readOnlyShm==0 || pWal->readOnlyShm==1 ); /* If the first page of the wal-index has been mapped, try to read the ** wal-index header immediately, without holding any lock. This usually @@ -1939,11 +1925,11 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ badHdr = (page0 ? walIndexTryHdr(pWal, pChanged) : 1); /* If the first attempt failed, it might have been due to a race - ** with a writer. So lock the WAL_WRITE_LOCK byte and try again. + ** with a writer. So get a WRITE lock and try again. */ assert( badHdr==0 || pWal->writeLock==0 ); if( badHdr ){ - if( pWal->readOnlyShm ){ + if( pWal->readOnly & WAL_SHM_RDONLY ){ if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; @@ -2150,7 +2136,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } /* There was once an "if" here. The extra "{" is to preserve indentation. */ { - if( pWal->readOnlyShm==0 && (mxReadMark < pWal->hdr.mxFrame || mxI==0) ){ + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkhdr.mxFrame || mxI==0) + ){ for(i=1; ireadOnlyShm && rc==SQLITE_OK) ); return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; } @@ -2221,18 +2208,13 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** Pager layer will use this to know that is cache is stale and ** needs to be flushed. */ -int sqlite3WalBeginReadTransaction(Wal *pWal, int readOnlyShm, int *pChanged){ +int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ int rc; /* Return code */ int cnt = 0; /* Number of TryBeginRead attempts */ - if( pWal->nWiData==0 || pWal->apWiData[0]==0 ){ - assert( readOnlyShm==0 || readOnlyShm==1 ); - pWal->readOnlyShm = readOnlyShm*2; - } do{ rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); }while( rc==WAL_RETRY ); - assert( rc || pWal->readOnlyShm==0 || (readOnlyShm && pWal->readOnlyShm==1) ); testcase( (rc&0xff)==SQLITE_BUSY ); testcase( (rc&0xff)==SQLITE_IOERR ); testcase( rc==SQLITE_PROTOCOL ); @@ -2407,7 +2389,6 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ if( pWal->readOnly ){ return SQLITE_READONLY; } - assert( pWal->readOnlyShm==0 ); /* Only one writer allowed at a time. Get the write lock. Return ** SQLITE_BUSY if unable. @@ -2814,11 +2795,8 @@ int sqlite3WalCheckpoint( assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); + if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); - if( pWal->readOnlyShm ){ - return SQLITE_READONLY; - } - rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ /* Usually this is SQLITE_BUSY meaning that another thread or process diff --git a/src/wal.h b/src/wal.h index df09beffe7..a62b23bbdc 100644 --- a/src/wal.h +++ b/src/wal.h @@ -23,7 +23,7 @@ # define sqlite3WalOpen(x,y,z) 0 # define sqlite3WalLimit(x,y) # define sqlite3WalClose(w,x,y,z) 0 -# define sqlite3WalBeginReadTransaction(x,y,z) 0 +# define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) # define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 @@ -60,7 +60,7 @@ void sqlite3WalLimit(Wal*, i64); ** write to or checkpoint the WAL. sqlite3WalCloseSnapshot() closes the ** transaction and releases the lock. */ -int sqlite3WalBeginReadTransaction(Wal *pWal, int, int *); +int sqlite3WalBeginReadTransaction(Wal *pWal, int *); void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ From fd019ef7ceaf17d41a47c3be1ef7309deb9be7eb Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Jun 2011 20:13:36 +0000 Subject: [PATCH 16/69] Make use of the sqlite3GetBoolean() interface for more robust processing of the readonly_shm query parameter inside of unixShmMap(). FossilOrigin-Name: 1f930d7e04cd4a5ff3d91a0e9f1b62114f1cebd2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8b6e11236e..b9fd02df11 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\swal-readonly\sbranch\sso\sthat\sit\sdoes\snot\srequire\schanges\sto\nanything\sother\sthan\sos_unix.c\sand\swal.c\sand\sa\scouple\sof\snew\serror\scodes. -D 2011-06-01T20:01:49.650 +C Make\suse\sof\sthe\ssqlite3GetBoolean()\sinterface\sfor\smore\srobust\sprocessing\nof\sthe\sreadonly_shm\squery\sparameter\sinside\sof\sunixShmMap(). +D 2011-06-01T20:13:36.269 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/os.c 22ac61d06e72a0dac900400147333b07b13d8e1d F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c a59c0718021934157f235468788c793cbc0d53de +F src/os_unix.c fd4e9588ff0ce09720721ce739ab2682202875ae F src/os_win.c 218b899469e570d46eb8147c2383075f7c026230 F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 0b63b71357a65e26ecd3f3bb34a5f14feee322f4 -R 20a045d8e6470f5f17b67b2f15ff8859 +P d6b4709de4d1f8af001f58938247f00a652a616e +R fdd310892b7cea51b352170f4a152593 U drh -Z f7b3b70c555e3fb5f3360ed8e60c2877 +Z cc8ff93f2b92f997226013b652879169 diff --git a/manifest.uuid b/manifest.uuid index 6f393899aa..203442f1d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6b4709de4d1f8af001f58938247f00a652a616e \ No newline at end of file +1f930d7e04cd4a5ff3d91a0e9f1b62114f1cebd2 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index e6d1774a49..5860ae71a9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3787,7 +3787,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ if( pShmNode->h<0 ){ const char *zRO; zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); - if( zRO && (zRO[0]!='0' || zRO[1]!=0) ){ + if( zRO && sqlite3GetBoolean(zRO) ){ pShmNode->h = robust_open(zShmFilename, O_RDONLY, (sStat.st_mode & 0777)); pShmNode->isReadonly = 1; From 070d422d31cec08996630f03622afed9b1d2407a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Jun 2011 15:48:51 +0000 Subject: [PATCH 17/69] Add a missing check for out-of-memory in the lemon code generator. FossilOrigin-Name: efb20b9da6c7cb310a449cc818eaccd3d5bb4ab3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lemon.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f2e67cf931..355ba1eb5b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sread-only\sshared\smemory\sbranch\sinto\strunk.\s\sAfter\sthis\smerge,\san\nunprivileged\sprocess\scan\sopen\sWAL-mode\sdatabases\sowned\sby\sanother\suser\sas\nlong\sas\sa\sdatabase\sconnection\swith\swrite\spermission\sexists\son\sthe\sdatabase\nfile\sand\sif\sthe\sreadonly_shm=1\sURI\squery\sparameter\sis\ssupplied. -D 2011-06-02T13:04:33.467 +C Add\sa\smissing\scheck\sfor\sout-of-memory\sin\sthe\slemon\scode\sgenerator. +D 2011-06-02T15:48:51.684 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -909,7 +909,7 @@ F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c dfd81a51b6e27e469ba21d01a75ddf092d429027 +F tool/lemon.c 2f182cf58a44a29107ad0027e4e696c79cbb9ad6 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P e704e8690ae35decc9769a45cf8d519ccad8b79d 1f930d7e04cd4a5ff3d91a0e9f1b62114f1cebd2 -R fdd310892b7cea51b352170f4a152593 +P 19084a6641f77a62110b04ea50e298fe132ea784 +R 35b0b86c3f244adcf3711ae5d8007435 U drh -Z c21bc600f09ce2bc60c48c66dd9bd252 +Z df920ad923d5c14dce2ff7d7b1f2c419 diff --git a/manifest.uuid b/manifest.uuid index 2a33332676..2ce7958fe9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19084a6641f77a62110b04ea50e298fe132ea784 \ No newline at end of file +efb20b9da6c7cb310a449cc818eaccd3d5bb4ab3 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 898022e28c..0a50524854 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -3434,6 +3434,10 @@ void print_stack_union( /* Allocate and initialize types[] and allocate stddt[] */ arraysize = lemp->nsymbol * 2; types = (char**)calloc( arraysize, sizeof(char*) ); + if( types==0 ){ + fprintf(stderr,"Out of memory.\n"); + exit(1); + } for(i=0; ivartype ){ @@ -3447,7 +3451,7 @@ void print_stack_union( if( len>maxdtlength ) maxdtlength = len; } stddt = (char*)malloc( maxdtlength*2 + 1 ); - if( types==0 || stddt==0 ){ + if( stddt==0 ){ fprintf(stderr,"Out of memory.\n"); exit(1); } From 5bf3934652568cdef29ca24ab9585004e11d0ce3 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Jun 2011 17:24:49 +0000 Subject: [PATCH 18/69] Fix a faulty assert() in the WAL-mode logic for read-only shared memory. FossilOrigin-Name: a13cfe616284f4ee86f0406e7b8fe8f9ba6e6990 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 355ba1eb5b..f04e836195 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\scheck\sfor\sout-of-memory\sin\sthe\slemon\scode\sgenerator. -D 2011-06-02T15:48:51.684 +C Fix\sa\sfaulty\sassert()\sin\sthe\sWAL-mode\slogic\sfor\sread-only\sshared\smemory. +D 2011-06-02T17:24:49.997 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 9ba8c7fdb7d39260c033a402f6032d3e7bc5d336 -F src/wal.c fd63d07233203dd3bd29cbe1ae5c8bb2c34e08fc +F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 19084a6641f77a62110b04ea50e298fe132ea784 -R 35b0b86c3f244adcf3711ae5d8007435 +P efb20b9da6c7cb310a449cc818eaccd3d5bb4ab3 +R 7af7e431287e95dc91be1106acfd740b U drh -Z df920ad923d5c14dce2ff7d7b1f2c419 +Z e77c10de1c037b59f0516366918c7ba0 diff --git a/manifest.uuid b/manifest.uuid index 2ce7958fe9..b636e3ed7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -efb20b9da6c7cb310a449cc818eaccd3d5bb4ab3 \ No newline at end of file +a13cfe616284f4ee86f0406e7b8fe8f9ba6e6990 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 9f9e1115fa..b9a03dff23 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2152,7 +2152,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } } if( mxI==0 ){ - assert( rc==SQLITE_BUSY ); + assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; } From e414854800a7d87c2a207774b5d65007e3592eb9 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Jun 2011 19:57:24 +0000 Subject: [PATCH 19/69] Changes to improve performance and support LIMIT clauses on fts3 tables. This branch is unstable for now. FossilOrigin-Name: 28149a7882a1e9dfe4a75ec5b91d176ebe6284e9 --- ext/fts3/fts3.c | 1120 +++++++++++++++++++++++++++++++++------ ext/fts3/fts3Int.h | 86 ++- ext/fts3/fts3_aux.c | 2 +- ext/fts3/fts3_expr.c | 5 +- ext/fts3/fts3_snippet.c | 110 ++-- ext/fts3/fts3_term.c | 2 +- ext/fts3/fts3_write.c | 170 +++++- manifest | 28 +- manifest.uuid | 2 +- test/fts3defer.test | 29 +- test/permutations.test | 1 + 11 files changed, 1274 insertions(+), 281 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 536c47fc78..a7f176a700 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -313,6 +313,9 @@ SQLITE_EXTENSION_INIT1 #endif +static char *fts3EvalPhrasePoslist(Fts3Phrase *, int *); +static sqlite3_int64 fts3EvalPhraseDocid(Fts3Phrase *); + /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. @@ -1207,14 +1210,16 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ */ if( pInfo->nOrderBy==1 ){ struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; - if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc==0 + && (pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1) + ){ if( pOrder->desc ){ pInfo->idxStr = "DESC"; }else{ pInfo->idxStr = "ASC"; } + pInfo->orderByConsumed = 1; } - pInfo->orderByConsumed = 1; } return SQLITE_OK; @@ -1256,6 +1261,8 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } +static int fts3RowidMethod(sqlite3_vtab_cursor *, sqlite3_int64*); + /* ** Position the pCsr->pStmt statement so that it is on the row ** of the %_content table that contains the last match. Return @@ -1263,8 +1270,8 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ if( pCsr->isRequireSeek ){ - pCsr->isRequireSeek = 0; sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ return SQLITE_OK; }else{ @@ -2220,7 +2227,7 @@ static int fts3DeferredTermSelect( ** Append SegReader object pNew to the end of the pCsr->apSegment[] array. */ static int fts3SegReaderCursorAppend( - Fts3SegReaderCursor *pCsr, + Fts3MultiSegReader *pCsr, Fts3SegReader *pNew ){ if( (pCsr->nSegment%16)==0 ){ @@ -2245,7 +2252,7 @@ static int fts3SegReaderCursor( int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3SegReaderCursor *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ int rc = SQLITE_OK; int rc2; @@ -2316,7 +2323,7 @@ int sqlite3Fts3SegReaderCursor( int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3SegReaderCursor *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ assert( iIndex>=0 && iIndexnIndex ); assert( iLevel==FTS3_SEGCURSOR_ALL @@ -2331,7 +2338,7 @@ int sqlite3Fts3SegReaderCursor( ** full-text tables. */ assert( isScan==0 || p->aIndex==0 ); - memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); + memset(pCsr, 0, sizeof(Fts3MultiSegReader)); return fts3SegReaderCursor( p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr @@ -2342,23 +2349,23 @@ static int fts3SegReaderCursorAddZero( Fts3Table *p, const char *zTerm, int nTerm, - Fts3SegReaderCursor *pCsr + Fts3MultiSegReader *pCsr ){ return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); } -static int fts3TermSegReaderCursor( +int sqlite3Fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ - Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ - Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ - pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); + pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); if( pSegcsr ){ int i; int nCost = 0; @@ -2371,6 +2378,7 @@ static int fts3TermSegReaderCursor( bFound = 1; rc = sqlite3Fts3SegReaderCursor( p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + pSegcsr->bLookup = 1; } } @@ -2391,6 +2399,7 @@ static int fts3TermSegReaderCursor( rc = sqlite3Fts3SegReaderCursor( p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr ); + pSegcsr->bLookup = !isPrefix; } for(i=0; rc==SQLITE_OK && inSegment; i++){ rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); @@ -2402,7 +2411,7 @@ static int fts3TermSegReaderCursor( return rc; } -static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } @@ -2427,7 +2436,7 @@ static int fts3TermSelect( char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ - Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ TermSelect tsc; /* Context object for fts3TermSelectCb() */ Fts3SegFilter filter; /* Segment term filter configuration */ @@ -2554,7 +2563,7 @@ static void fts3DoclistStripPositions( } } -/* +/* ** Return a DocList corresponding to the phrase *pPhrase. ** ** If this function returns SQLITE_OK, but *pnOut is set to a negative value, @@ -2582,26 +2591,6 @@ static int fts3PhraseSelect( int iPrevTok = 0; int nDoc = 0; - /* If this is an xFilter() evaluation, create a segment-reader for each - ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo - ** evaluation, only create segment-readers if there are no Fts3DeferredToken - ** objects attached to the phrase-tokens. - */ - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pSegcsr==0 ){ - if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) - || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) - || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) - ){ - rc = fts3TermSegReaderCursor( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr - ); - if( rc!=SQLITE_OK ) return rc; - } - } - } - for(ii=0; iinToken; ii++){ Fts3PhraseToken *pTok; /* Token to find doclist for */ int iTok = 0; /* The token being queried this iteration */ @@ -2626,7 +2615,7 @@ static int fts3PhraseSelect( /* Find the remaining token with the lowest cost. */ for(jj=0; jjnToken; jj++){ - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr; + Fts3MultiSegReader *pSegcsr = pPhrase->aToken[jj].pSegcsr; if( pSegcsr && pSegcsr->nCostnCost; @@ -2828,13 +2817,13 @@ static int fts3ExprAllocateSegReaders( } if( pExpr->eType==FTSQUERY_PHRASE ){ + int ii; /* Used to iterate through phrase tokens */ Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; for(ii=0; rc==SQLITE_OK && iinToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; if( pTok->pSegcsr==0 ){ - rc = fts3TermSegReaderCursor( + rc = sqlite3Fts3TermSegReaderCursor( pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr ); } @@ -2880,7 +2869,7 @@ static int fts3ExprCost(Fts3Expr *pExpr){ int ii; nCost = 0; for(ii=0; iinToken; ii++){ - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr; + Fts3MultiSegReader *pSegcsr = pPhrase->aToken[ii].pSegcsr; if( pSegcsr ) nCost += pSegcsr->nCost; } }else{ @@ -3175,34 +3164,38 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ int rc = SQLITE_OK; /* Return code */ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - pCsr->eEvalmode = FTS3_EVAL_NEXT; - do { - if( pCsr->aDoclist==0 ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - break; - } - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); - }else{ - if( pCsr->desc==0 ){ - if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ + if( pCsr->bIncremental ){ + rc = sqlite3Fts3EvalNext(pCsr, pCsr->pExpr); + }else{ + pCsr->eEvalmode = FTS3_EVAL_NEXT; + do { + if( pCsr->aDoclist==0 ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); break; } - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); }else{ - fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId); - if( pCsr->pNextId<=pCsr->aDoclist ){ - pCsr->isEof = 1; - break; + if( pCsr->desc==0 ){ + if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ + pCsr->isEof = 1; + break; + } + fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); + }else{ + fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId); + if( pCsr->pNextId<=pCsr->aDoclist ){ + pCsr->isEof = 1; + break; + } } + sqlite3_reset(pCsr->pStmt); + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; } - sqlite3_reset(pCsr->pStmt); - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - } - }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); + }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); + } return rc; } @@ -3230,11 +3223,7 @@ static int fts3FilterMethod( int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - const char *azSql[] = { - "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ - "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */ - }; - int rc; /* Return code */ + int rc; char *zSql; /* SQL statement used to access %_content */ Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; @@ -3266,8 +3255,8 @@ static int fts3FilterMethod( ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ - p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]", - zQuery); + static const char *zErr = "malformed MATCH expression: [%s]"; + p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); } return rc; } @@ -3275,7 +3264,9 @@ static int fts3FilterMethod( rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); + pCsr->bIncremental = 1; + rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); + sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; @@ -3287,23 +3278,26 @@ static int fts3FilterMethod( ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ - zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH]; - zSql = sqlite3_mprintf( - zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC") - ); - if( !zSql ){ - rc = SQLITE_NOMEM; + if( idxNum==FTS3_FULLSCAN_SEARCH ){ + const char *zSort = (idxStr ? idxStr : "ASC"); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); } - if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK ) return rc; + + if( idxNum==FTS3_DOCID_SEARCH ){ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + if( rc!=SQLITE_OK ) return rc; } - pCsr->eSearch = (i16)idxNum; assert( pCsr->desc==0 ); - if( rc!=SQLITE_OK ) return rc; + pCsr->eSearch = (i16)idxNum; if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){ sqlite3_int64 iDocid = 0; char *csr = pCsr->aDoclist; @@ -3337,7 +3331,9 @@ static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - if( pCsr->aDoclist ){ + if( pCsr->bIncremental ){ + *pRowid = sqlite3Fts3EvalDocid(pCsr, pCsr->pExpr); + }else if( pCsr->aDoclist ){ *pRowid = pCsr->iPrevId; }else{ /* This branch runs if the query is implemented using a full-table scan @@ -3460,11 +3456,13 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ ** functions. */ int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc; - Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->eType==FTSQUERY_PHRASE && pPhrase ); - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - rc = fts3EvalExpr(pCsr, pExpr, &pPhrase->aDoclist, &pPhrase->nDoclist, 1); + int rc = SQLITE_OK; + if( pCsr->bIncremental==0 ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->eType==FTSQUERY_PHRASE && pPhrase ); + assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); + rc = fts3EvalExpr(pCsr, pExpr, &pPhrase->aDoclist, &pPhrase->nDoclist, 1); + } return rc; } @@ -3480,9 +3478,10 @@ int sqlite3Fts3ExprLoadFtDoclist( char **paDoclist, int *pnDoclist ){ - int rc; - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); + int rc = SQLITE_OK; assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); + assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); + assert( pCsr->bIncremental==0 ); pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); pCsr->eEvalmode = FTS3_EVAL_NEXT; @@ -3507,82 +3506,6 @@ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ *ppPoslist = p; } - -/* -** After ExprLoadDoclist() (see above) has been called, this function is -** used to iterate/search through the position lists that make up the doclist -** stored in pExpr->aDoclist. -*/ -char *sqlite3Fts3FindPositions( - Fts3Cursor *pCursor, /* Associate FTS3 cursor */ - Fts3Expr *pExpr, /* Access this expressions doclist */ - sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ - int iCol /* Column of requested pos-list */ -){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pPhrase->isLoaded ); - - if( pPhrase->aDoclist ){ - char *pEnd = &pPhrase->aDoclist[pPhrase->nDoclist]; - char *pCsr; - - if( pPhrase->pCurrent==0 ){ - if( pCursor->desc==0 ){ - pPhrase->pCurrent = pPhrase->aDoclist; - pPhrase->iCurrent = 0; - fts3GetDeltaVarint(&pPhrase->pCurrent, &pPhrase->iCurrent); - }else{ - pCsr = pPhrase->aDoclist; - while( pCsriCurrent); - fts3PoslistCopy(0, &pCsr); - } - fts3ReversePoslist(pPhrase->aDoclist, &pCsr); - pPhrase->pCurrent = pCsr; - } - } - pCsr = pPhrase->pCurrent; - assert( pCsr ); - - while( (pCursor->desc==0 && pCsrdesc && pCsr>pPhrase->aDoclist) - ){ - if( pCursor->desc==0 && pPhrase->iCurrentiCurrent); - } - pPhrase->pCurrent = pCsr; - }else if( pCursor->desc && pPhrase->iCurrent>iDocid ){ - fts3GetReverseDeltaVarint(&pCsr, pPhrase->aDoclist, &pPhrase->iCurrent); - fts3ReversePoslist(pPhrase->aDoclist, &pCsr); - pPhrase->pCurrent = pCsr; - }else{ - if( pPhrase->iCurrent==iDocid ){ - int iThis = 0; - if( iCol<0 ){ - /* If iCol is negative, return a pointer to the start of the - ** position-list (instead of a pointer to the start of a list - ** of offsets associated with a specific column). - */ - return pCsr; - } - while( iThiseType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = sqlite3Fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + }else{ + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pRc); + } + } +} + +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, + Fts3Phrase *p +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + char *aDoclist = 0; + int nDoclist = 0; + int iPrev = -1; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pSegcsr || pToken->pDeferred ); + + if( pToken->pDeferred==0 ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); + if( rc==SQLITE_OK ){ + if( pThis==0 ){ + sqlite3_free(aDoclist); + aDoclist = 0; + nDoclist = 0; + break; + }else if( aDoclist==0 ){ + aDoclist = pThis; + nDoclist = nThis; + }else{ + assert( iPrev>=0 ); + fts3DoclistMerge(MERGE_POS_PHRASE, iToken-iPrev, + 0, pThis, &nThis, aDoclist, nDoclist, pThis, nThis, 0 + ); + sqlite3_free(aDoclist); + aDoclist = pThis; + nDoclist = nThis; + } + iPrev = iToken; + } + } + } + + if( rc==SQLITE_OK ){ + p->doclist.aAll = aDoclist; + p->doclist.nAll = nDoclist; + }else{ + sqlite3_free(aDoclist); + } + return rc; +} + +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + int nMaxUndeferred = -1; + char *aPoslist = 0; + int nPoslist = 0; + int iPrev = -1; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + assert( iPrev>=0 ); + + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = aOut - aPoslist; + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + }else{ + nMaxUndeferred = iToken; + } + } + + if( iPrev>=0 ){ + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); + sqlite3_free(aPoslist); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + } + } + + return SQLITE_OK; +} + + +/* +** The following three functions: +** +** fts3EvalPhraseStart() +** fts3EvalPhraseNext() +** fts3EvalPhraseReset() +** +** May be used with a phrase object after fts3EvalAllocateReaders() has been +** called to iterate through the set of docids that match the phrase. +** +** After a successful call to fts3EvalPhraseNext(), the following two +** functions may be called to access the current docid and position-list. +** +** fts3EvalPhraseDocid() +** fts3EvalPhrasePoslist() +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + int rc; + Fts3Doclist *pList = &p->doclist; + Fts3PhraseToken *pFirst = &p->aToken[0]; + + assert( pList->aAll==0 ); + + if( p->nToken==1 && bOptOk==1 + && pFirst->pSegcsr && pFirst->pSegcsr->bLookup + ){ + /* Use the incremental approach. */ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + rc = sqlite3Fts3MsrIncrStart( + pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); + p->bIncr = 1; + + }else{ + + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +*/ +static int fts3EvalPhraseNext(Fts3Cursor *pCsr, Fts3Phrase *p, u8 *pbEof){ + int rc = SQLITE_OK; + + if( p->bIncr ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + assert( p->nToken==1 ); + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &p->doclist.iDocid, &p->doclist.pList, &p->doclist.nList + ); + if( rc==SQLITE_OK && !p->doclist.pList ){ + *pbEof = 1; + } + }else{ + char *pIter; + Fts3Doclist *pDL = &p->doclist; + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; + } + + if( pIter>=&pDL->aAll[pDL->nAll] ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + fts3GetDeltaVarint(&pIter, &pDL->iDocid); + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (pIter - pDL->pList); + pDL->pNextDocid = pIter; + *pbEof = 0; + } + } + + return rc; +} + +static int fts3EvalPhraseReset(Fts3Cursor *pCsr, Fts3Phrase *p){ + return SQLITE_OK; +} + +static sqlite3_int64 fts3EvalPhraseDocid(Fts3Phrase *p){ + return p->doclist.iDocid; +} + +static char *fts3EvalPhrasePoslist(Fts3Phrase *p, int *pnList){ + if( pnList ){ + *pnList = p->doclist.nList; + } + return p->doclist.pList; +} + +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int bOptOk, + int *pRc +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); + }else{ + if( pExpr->eType==FTSQUERY_NEAR ){ + bOptOk = 0; + } + fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +static void fts3EvalNearMerge( + Fts3Expr *p1, + Fts3Expr *p2, + int nNear, + int *pRc +){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + Fts3Phrase *pLeft = p1->pPhrase; + Fts3Phrase *pRight = p2->pPhrase; + + assert( p2->eType==FTSQUERY_PHRASE && pLeft ); + assert( p2->eType==FTSQUERY_PHRASE && pRight ); + + if( pLeft->doclist.aAll==0 ){ + sqlite3_free(pRight->doclist.aAll); + pRight->doclist.aAll = 0; + pRight->doclist.nAll = 0; + }else if( pRight->doclist.aAll ){ + char *aOut; /* Buffer in which to assemble new doclist */ + int nOut; /* Size of buffer aOut in bytes */ + + *pRc = fts3NearMerge(MERGE_POS_NEAR, nNear, + pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll, + pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll, + &aOut, &nOut + ); + sqlite3_free(pRight->doclist.aAll); + pRight->doclist.aAll = aOut; + pRight->doclist.nAll = nOut; + } + } +} + +static void fts3EvalNearTrim(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ + + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_NEAR ){ + Fts3Expr *pLeft = pExpr->pLeft; + int nPhrase = 2; + Fts3Expr **aPhrase; + + assert( pLeft ); + assert( pExpr->pRight ); + assert( pExpr->pRight->eType==FTSQUERY_PHRASE ); + + while( pLeft->eType!=FTSQUERY_PHRASE ){ + assert( pLeft->eType==FTSQUERY_NEAR ); + assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); + pLeft = pLeft->pLeft; + nPhrase++; + } + + aPhrase = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nPhrase); + if( !aPhrase ){ + *pRc = SQLITE_NOMEM; + }else{ + int i = 1; + aPhrase[0] = pLeft; + do { + pLeft = pLeft->pParent; + aPhrase[i++] = pLeft->pRight; + }while( pLeft!=pExpr ); + + for(i=0; i<(nPhrase-1); i++){ + int nNear = aPhrase[i+1]->pParent->nNear; + fts3EvalNearMerge(aPhrase[i], aPhrase[i+1], nNear, pRc); + } + for(i=nPhrase-2; i>=0; i--){ + int nNear = aPhrase[i+1]->pParent->nNear; + fts3EvalNearMerge(aPhrase[i+1], aPhrase[i], nNear, pRc); + } + + sqlite3_free(aPhrase); + } + + }else{ + fts3EvalNearTrim(pCsr, pExpr->pLeft, pRc); + fts3EvalNearTrim(pCsr, pExpr->pRight, pRc); + } + } +} + +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3PhraseToken *pToken; + int nOvfl; + int iCol; +}; + +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + Fts3TokenAndCost **ppTC, + int *pRc +){ + if( *pRc==SQLITE_OK && pExpr ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType==FTSQUERY_AND ){ + fts3EvalTokenCosts(pCsr, pExpr->pLeft, ppTC, pRc); + fts3EvalTokenCosts(pCsr, pExpr->pRight, ppTC, pRc); + } + } +} + +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + int rc; + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + assert( a ); + + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + + *pnPage = pCsr->nRowAvg; + return SQLITE_OK; +} + +int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &rc); + + /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: + ** This call will eventually also be responsible for determining which + ** tokens are 'deferred' until the document text is loaded into memory. + ** + ** Each token in each phrase is dealt with using one of the following + ** three strategies: + ** + ** 1. Entire doclist loaded into memory as part of the + ** fts3EvalStartReaders() call. + ** + ** 2. Doclist loaded into memory incrementally, as part of each + ** sqlite3Fts3EvalNext() call. + ** + ** 3. Token doclist is never loaded. Instead, documents are loaded into + ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() + ** call. This is known as a "deferred" token. + */ + + /* If bOptOk is true, check if there are any tokens that can be + ** deferred (strategy 3). */ + if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ + Fts3TokenAndCost *aTC; + aTC = (Fts3TokenAndCost *)sqlite3_malloc(sizeof(Fts3TokenAndCost) * nToken); + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + int nDocEst = 0; + int nDocSize; + Fts3TokenAndCost *pTC = aTC; + + rc = fts3EvalAverageDocsize(pCsr, &nDocSize); + fts3EvalTokenCosts(pCsr, pExpr, &pTC, &rc); + nToken = pTC-aTC; + + for(ii=0; rc==SQLITE_OK && iinOvfl) ){ + pTC = &aTC[jj]; + } + } + assert( pTC ); + + /* At this point pTC points to the cheapest remaining token. */ + if( ii==0 ){ + if( pTC->nOvfl ){ + nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; + }else{ + /* TODO: Fix this so that the doclist need not be read twice. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + if( rc==SQLITE_OK ){ + nDocEst = fts3DoclistCountDocids(1, pList, nList); + } + sqlite3_free(pList); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + } + } + }else{ + if( pTC->nOvfl>=(nDocEst*nDocSize) ){ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + } + nDocEst = 1 + (nDocEst/4); + } + pTC->pToken = 0; + } + sqlite3_free(aTC); + } + } + + fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc); + + /* Fix the results of NEAR expressions. */ + fts3EvalNearTrim(pCsr, pExpr, &rc); + + return rc; +} + +static void fts3EvalNext( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( *pRc==SQLITE_OK ){ + + pExpr->bStart = 1; + switch( pExpr->eType ){ + + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + assert( !pLeft->bDeferred || !pRight->bDeferred ); + if( pLeft->bDeferred ){ + fts3EvalNext(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + fts3EvalNext(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + int iDiff = pLeft->iDocid - pRight->iDocid; + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else{ + fts3EvalNext(pCsr, pRight, pRc); + } + } + + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pLeft->iDocid==pRight->iDocid ){ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + }else if( + pRight->bEof || (pLeft->bEof==0 && pLeft->iDocidiDocid) + ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else{ + fts3EvalNext(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + if( pRight->bEof || (pLeft->bEof==0 && pLeft->iDocidiDocid) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNext(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + do { + fts3EvalNext(pCsr, pLeft, pRc); + if( pLeft->bEof ) break; + while( !*pRc && !pRight->bEof && pRight->iDocidiDocid ){ + fts3EvalNext(pCsr, pRight, pRc); + } + }while( !pRight->bEof && pRight->iDocid==pLeft->iDocid && !*pRc ); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: + assert( pExpr->eType==FTSQUERY_PHRASE ); + *pRc = fts3EvalPhraseNext(pCsr, pExpr->pPhrase, &pExpr->bEof); + pExpr->iDocid = fts3EvalPhraseDocid(pExpr->pPhrase); + break; + } + } +} + +static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ + int bHit = 0; + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + ); + break; + + case FTSQUERY_OR: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + || fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + ); + break; + + case FTSQUERY_NOT: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + ); + break; + + default: + assert( pExpr->eType==FTSQUERY_PHRASE ); + *pRc = fts3EvalDeferredPhrase(pCsr, pExpr->pPhrase); + bHit = (pExpr->pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + break; + } + } + return bHit; +} + +/* +** Return 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does not match the query. +*/ +static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK && pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + sqlite3Fts3FreeDeferredDoclists(pCsr); + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the expression passed as an +** argument. +*/ +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + int rc = SQLITE_OK; /* Return Code */ + assert( pCsr->isEof==0 ); + assert( pCsr->bIncremental ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + sqlite3_reset(pCsr->pStmt); + fts3EvalNext(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); + } + return rc; +} + +int sqlite3Fts3EvalFinish(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + return SQLITE_OK; +} + +sqlite3_int64 sqlite3Fts3EvalDocid(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + return pExpr->iDocid; +} + +/* +** Return a pointer to the entire doclist, including positions, associated +** with the phrase passed as the second argument. +*/ +int sqlite3Fts3EvalPhraseDoclist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + const char **ppList, /* OUT: Buffer containing doclist */ + int *pnList /* OUT: Size of returned buffer, in bytes */ +){ + int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase->bIncr ){ + /* This phrase was being loaded from disk incrementally. But the + ** matchinfo() function requires that the entire doclist be loaded into + ** memory. This block loads the doclist into memory and modifies the + ** Fts3Phrase structure so that it does not use the incremental strategy. + */ + TESTONLY( int bEof = pExpr->bEof; ) + TESTONLY( int bStart = pExpr->bStart; ) + sqlite3_int64 iDocid = pExpr->iDocid; + + sqlite3Fts3EvalPhraseCleanup(pPhrase); + pExpr->iDocid = 0; + + rc = sqlite3Fts3EvalStart(pCsr, pExpr, 0); + assert( pExpr->bEof==bEof ); + assert( pExpr->bStart==bStart ); + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + if( pExpr->bStart && !pExpr->bEof ){ + pExpr->bStart = 0; + while( rc==SQLITE_OK && pExpr->bEof==0 && pExpr->iDocid!=iDocid ){ + fts3EvalNext(pCsr, pExpr, &rc); + } + } + } + + *pnList = pPhrase->doclist.nAll; + *ppList = pPhrase->doclist.aAll; + return rc; +} + +char *sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + sqlite3_int64 iDocid, /* Docid to return position list for */ + int iCol /* Column to return position list for */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter = pPhrase->doclist.pList; + int iThis; + + assert( iCol>=0 && iColnColumn ); + if( !pIter + || pExpr->bEof + || pExpr->iDocid!=iDocid + || (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) + ){ + return 0; + } + + assert( pPhrase->doclist.nList>0 ); + if( *pIter==0x01 ){ + pIter++; + pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist.aAll); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } +} + #endif diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 42be10773a..0d69029dfb 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -47,6 +47,11 @@ */ #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif + /* ** Maximum length of a varint encoded integer. The varint format is different ** from that used by SQLite, so the maximum length is 10, not 9. @@ -142,10 +147,11 @@ typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3PhraseToken Fts3PhraseToken; +typedef struct Fts3Doclist Fts3Doclist; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3SegReaderCursor Fts3SegReaderCursor; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; /* ** A connection to a fulltext index is an instance of the following @@ -224,6 +230,7 @@ struct Fts3Cursor { u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ + int bIncremental; /* True to use incremental querying */ int nPhrase; /* Number of matchable phrases in query */ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ @@ -263,6 +270,17 @@ struct Fts3Cursor { #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + + sqlite3_int64 iDocid; /* Current docid (if p!=0) */ + char *pNextDocid; /* Pointer to next docid */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +} doclist; + /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. @@ -277,23 +295,29 @@ struct Fts3PhraseToken { /* Variables above this point are populated when the expression is ** parsed (by code in fts3_expr.c). Below this point the variables are ** used when evaluating the expression. */ - int bFulltext; /* True if full-text index was used */ - Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ }; struct Fts3Phrase { - /* Variables populated by fts3_expr.c when parsing a MATCH expression */ - int nToken; /* Number of tokens in the phrase */ - int iColumn; /* Index of column this phrase must match */ + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ +#if 1 int isLoaded; /* True if aDoclist/nDoclist are initialized. */ char *aDoclist; /* Buffer containing doclist */ int nDoclist; /* Size of aDoclist in bytes */ sqlite3_int64 iCurrent; char *pCurrent; +#endif + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ }; @@ -317,6 +341,12 @@ struct Fts3Expr { Fts3Expr *pLeft; /* Left operand */ Fts3Expr *pRight; /* Right operand */ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ }; /* @@ -366,11 +396,12 @@ void sqlite3Fts3SegmentsClose(Fts3Table *); #define FTS3_SEGCURSOR_PENDING -1 #define FTS3_SEGCURSOR_ALL -2 -int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); -int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); -void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); +int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); + int sqlite3Fts3SegReaderCursor( - Fts3Table *, int, int, const char *, int, int, int, Fts3SegReaderCursor *); + Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -387,7 +418,7 @@ struct Fts3SegFilter { int flags; }; -struct Fts3SegReaderCursor { +struct Fts3MultiSegReader { /* Used internally by sqlite3Fts3SegReaderXXX() calls */ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ int nSegment; /* Size of apSegment array */ @@ -396,8 +427,11 @@ struct Fts3SegReaderCursor { char *aBuffer; /* Buffer to merge doclists in */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ - /* Cost of running this iterator. Used by fts3.c only. */ - int nCost; + int iColFilter; /* If >=0, filter for this column */ + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ char *zTerm; /* Pointer to term buffer */ @@ -413,7 +447,6 @@ int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); -char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int); int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); @@ -446,4 +479,29 @@ int sqlite3Fts3InitTerm(sqlite3 *db); /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); +int sqlite3Fts3TermSegReaderCursor( + Fts3Cursor *pCsr, /* Virtual table cursor handle */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ +); + +int sqlite3Fts3EvalPhraseCache(Fts3Cursor *, Fts3Phrase *); +sqlite3_int64 sqlite3Fts3EvalDocid(Fts3Cursor *, Fts3Expr *); +int sqlite3Fts3EvalPhraseDoclist(Fts3Cursor*, Fts3Expr*, const char**,int*); +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr); +int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +char *sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *, Fts3Expr *, sqlite3_int64, int iCol); +int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); + +int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); + #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 5c7c7b0b0f..ded0202731 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -28,7 +28,7 @@ struct Fts3auxTable { struct Fts3auxCursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3SegReaderCursor csr; /* Must be right after "base" */ + Fts3MultiSegReader csr; /* Must be right after "base" */ Fts3SegFilter filter; char *zStop; int nStop; /* Byte-length of string zStop */ diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 095841d142..44636ed3fc 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -768,7 +768,10 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){ assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); - if( p->pPhrase ) sqlite3_free(p->pPhrase->aDoclist); + if( p->pPhrase ){ + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->pPhrase->aDoclist); + } sqlite3_free(p); } } diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index b31a1cf359..187ecbb57f 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -416,7 +416,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ pPhrase->nToken = pExpr->pPhrase->nToken; - pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->pCsr->iPrevId,p->iCol); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; @@ -826,46 +826,31 @@ static int fts3ExprGlobalHitsCb( void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; - Fts3Cursor *pCsr = p->pCursor; - Fts3Phrase *pPhrase = pExpr->pPhrase; - char *pIter; - char *pEnd; - char *pFree = 0; u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; - assert( pPhrase->isLoaded ); - - if( pCsr->pDeferred ){ - int ii; - for(ii=0; iinToken; ii++){ - if( pPhrase->aToken[ii].bFulltext ) break; - } - if( iinToken ){ - int nFree = 0; - int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); - if( rc!=SQLITE_OK ) return rc; - pIter = pFree; - pEnd = &pFree[nFree]; - }else{ - int iCol; /* Column index */ - for(iCol=0; iColnCol; iCol++){ - aOut[iCol*3 + 1] = (u32)p->nDoc; - aOut[iCol*3 + 2] = (u32)p->nDoc; - } - return SQLITE_OK; + if( pExpr->bDeferred ){ + int iCol; /* Column index */ + for(iCol=0; iColnCol; iCol++){ + aOut[iCol*3 + 1] = (u32)p->nDoc; + aOut[iCol*3 + 2] = (u32)p->nDoc; } }else{ - pIter = pPhrase->aDoclist; - pEnd = &pPhrase->aDoclist[pPhrase->nDoclist]; + char *pIter; + char *pEnd; + int n; + int rc = sqlite3Fts3EvalPhraseDoclist( + p->pCursor, pExpr, (const char **)&pIter, &n + ); + if( rc!=SQLITE_OK ) return rc; + pEnd = &pIter[n]; + + /* Fill in the global hit count matrix row for this phrase. */ + while( pIternCol * 3; int i; + sqlite3_int64 iDocid = p->pCursor->iPrevId; - for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; - - if( pExpr->pPhrase->aDoclist ){ + for(i=0; inCol; i++){ char *pCsr; - - pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, iDocid, i); if( pCsr ){ - fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0); + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; } } @@ -976,9 +961,8 @@ static int fts3MatchinfoSelectDoctotal( typedef struct LcsIterator LcsIterator; struct LcsIterator { Fts3Expr *pExpr; /* Pointer to phrase expression */ - char *pRead; /* Cursor used to iterate through aDoclist */ int iPosOffset; /* Tokens count up to end of this phrase */ - int iCol; /* Current column number */ + char *pRead; /* Cursor used to iterate through aDoclist */ int iPos; /* Current position */ }; @@ -1009,17 +993,10 @@ static int fts3LcsIteratorAdvance(LcsIterator *pIter){ int rc = 0; pRead += sqlite3Fts3GetVarint(pRead, &iRead); - if( iRead==0 ){ - pIter->iCol = LCS_ITERATOR_FINISHED; + if( iRead==0 || iRead==1 ){ + pRead = 0; rc = 1; }else{ - if( iRead==1 ){ - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - pIter->iCol = (int)iRead; - pIter->iPos = pIter->iPosOffset; - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - rc = 1; - } pIter->iPos += (int)(iRead-2); } @@ -1043,6 +1020,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ int i; int iCol; int nToken = 0; + sqlite3_int64 iDocid = pCsr->iPrevId; /* Allocate and populate the array of LcsIterator objects. The array ** contains one element for each matchable phrase in the query. @@ -1051,42 +1029,34 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ if( !aIter ) return SQLITE_NOMEM; memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; nToken -= pIter->pExpr->pPhrase->nToken; pIter->iPosOffset = nToken; - pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1); - if( pIter->pRead ){ - pIter->iPos = pIter->iPosOffset; - fts3LcsIteratorAdvance(&aIter[i]); - }else{ - pIter->iCol = LCS_ITERATOR_FINISHED; - } } for(iCol=0; iColnCol; iCol++){ int nLcs = 0; /* LCS value for this column */ int nLive = 0; /* Number of iterators in aIter not at EOF */ - /* Loop through the iterators in aIter[]. Set nLive to the number of - ** iterators that point to a position-list corresponding to column iCol. - */ for(i=0; inPhrase; i++){ - assert( aIter[i].iCol>=iCol ); - if( aIter[i].iCol==iCol ) nLive++; + LcsIterator *pIt = &aIter[i]; + pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iDocid, iCol); + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + nLive++; + } } - /* The following loop runs until all iterators in aIter[] have finished - ** iterating through positions in column iCol. Exactly one of the - ** iterators is advanced each time the body of the loop is run. - */ while( nLive>0 ){ LcsIterator *pAdv = 0; /* The iterator to advance by one position */ int nThisLcs = 0; /* LCS for the current iterator positions */ for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; - if( iCol!=pIter->iCol ){ + if( pIter->pRead==0 ){ /* This iterator is already at EOF for this column. */ nThisLcs = 0; }else{ @@ -1426,7 +1396,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iPos = 0; /* First position in position-list */ UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol); + pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iDocid, p->iCol); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index d7aa66855a..eff65b7bd1 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -33,7 +33,7 @@ struct Fts3termTable { struct Fts3termCursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3SegReaderCursor csr; /* Must be right after "base" */ + Fts3MultiSegReader csr; /* Must be right after "base" */ Fts3SegFilter filter; int isEof; /* True if cursor is at EOF */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index f664dec8cf..deeff3c418 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1155,6 +1155,8 @@ int sqlite3Fts3SegReaderCost( int nCost = 0; /* Cost in bytes to return */ int pgsz = p->nPgsz; /* Database page size */ + assert( pgsz>0 ); + /* If this seg-reader is reading the pending-terms table, or if all data ** for the segment is stored on the root page of the b-tree, then the cost ** is zero. In this case all required data is already in main memory. @@ -1223,6 +1225,40 @@ int sqlite3Fts3SegReaderCost( return rc; } +int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl +){ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; + + assert( p->bHasStat ); + assert( pgsz>0 ); + + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + int jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } + } + } + } + *pnOvfl = nOvfl; + return rc; +} + /* ** Free all allocations associated with the iterator passed as the ** second argument. @@ -2140,9 +2176,107 @@ static void fts3ColumnFilter( *pnList = nList; } +int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int nSegment = pCsr->nSegment; + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + for(i=0; iapSegment[i]; + do { + int rc = fts3SegReaderNext(p, pSeg); + if( rc!=SQLITE_OK ) return rc; + }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); + } + fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp); + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + fts3SegReaderFirstDocid(pCsr->apSegment[i]); + } + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +int sqlite3Fts3MsrIncrNext( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int rc = SQLITE_OK; + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } + + while( 1 ){ + Fts3SegReader *pSeg; + fts3SegReaderSort(pMsr->apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp); + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(apSegment[j], 0, 0); + } + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); + } + + if( nList>0 ){ + *piDocid = iDocid; + *paPoslist = pList; + *pnPoslist = nList; + break; + } + } + } + + return rc; +} + int sqlite3Fts3SegReaderStart( Fts3Table *p, /* Virtual table handle */ - Fts3SegReaderCursor *pCsr, /* Cursor object */ + Fts3MultiSegReader *pCsr, /* Cursor object */ Fts3SegFilter *pFilter /* Restrictions on range of iteration */ ){ int i; @@ -2173,7 +2307,7 @@ int sqlite3Fts3SegReaderStart( int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ - Fts3SegReaderCursor *pCsr /* Cursor object */ + Fts3MultiSegReader *pCsr /* Cursor object */ ){ int rc = SQLITE_OK; @@ -2308,8 +2442,9 @@ int sqlite3Fts3SegReaderStep( return rc; } + void sqlite3Fts3SegReaderFinish( - Fts3SegReaderCursor *pCsr /* Cursor object */ + Fts3MultiSegReader *pCsr /* Cursor object */ ){ if( pCsr ){ int i; @@ -2342,7 +2477,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ int iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ - Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ assert( iLevel==FTS3_SEGCURSOR_ALL @@ -2746,6 +2881,33 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ return rc; } +int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} + /* ** Add an entry for token pToken to the pCsr->pDeferred list. */ diff --git a/manifest b/manifest index 950ee87f2e..499805878d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schanges\smade\swhile\splanning\sa\slarger\schange. -D 2011-05-28T15:57:40.694 +C Changes\sto\simprove\sperformance\sand\ssupport\sLIMIT\sclauses\son\sfts3\stables.\sThis\sbranch\sis\sunstable\sfor\snow. +D 2011-06-02T19:57:24.733 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c eb59cdd89e9309ab9b2dca196a7c52f9e8927319 +F ext/fts3/fts3.c f92b6e0241a77a715d30dbeffd7c901053dbfda4 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 832f4d421f03a9d364186e779ee51994df500c62 -F ext/fts3/fts3_aux.c 2817a1ec9ffbad673cb1a1527ad807811bc7645b -F ext/fts3/fts3_expr.c 25e30cf24198333f2ed545af905b168e88f56903 +F ext/fts3/fts3Int.h ab1489076e7d54714d20bbbc7aaef8e694a7db50 +F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f +F ext/fts3/fts3_expr.c 5c789c744f98a007512f49d9cda4d2bb4cd56517 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c 6ee626017ddcf7d72ca4f587724e3506434fc0d7 -F ext/fts3/fts3_term.c 0ade1812c0e97f394b58299810dfd5d2fb7ba501 +F ext/fts3/fts3_snippet.c 10e0b0995ec82a2d93fbdf3159641cdf30f3c274 +F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c ed58c53fbcbc2ea79258e734159a5951ffeb1bd4 +F ext/fts3/fts3_write.c b145547430af9451f81cfed92fb7065da2efd870 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -463,7 +463,7 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 -F test/fts3defer.test d6cb0db9b5997ecf863d96ff419f83f8f2c87f4f +F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35 F test/fts3defer2.test 288bef6de15557319b8c12d476ebdc83688ef96c F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c @@ -612,7 +612,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 5b2a4cb756ffb2407cb4743163668d1d769febb6 +F test/permutations.test d27eac16dae111ff7cec331dab4bca08625ba65a F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P cc83991caae7c7d647432d5711b6cd80228c3002 -R eef52aec1faead4921ebaf39c1605d2b +P 84097a4c759b1d65890af885f137d3cb16eef584 +R e38e0cc84edbfdb5eae395b3be2f7a86 U dan -Z 01d8f7aea2ce8ebb35c05ccc519b614f +Z 53d59cf3dcd8991c66b0afd5fb898b1a diff --git a/manifest.uuid b/manifest.uuid index 6768a6838c..9e0b1f8e4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84097a4c759b1d65890af885f137d3cb16eef584 \ No newline at end of file +28149a7882a1e9dfe4a75ec5b91d176ebe6284e9 \ No newline at end of file diff --git a/test/fts3defer.test b/test/fts3defer.test index 1c9056fd9d..4bc0b0a7c3 100644 --- a/test/fts3defer.test +++ b/test/fts3defer.test @@ -20,6 +20,8 @@ ifcapable !fts3 { set sqlite_fts3_enable_parentheses 1 +set fts3_simple_deferred_tokens_only 1 + set ::testprefix fts3defer #-------------------------------------------------------------------------- @@ -257,7 +259,6 @@ foreach {tn setup} { do_select_test 1.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh' } {100} -if {$tn==3} breakpoint do_select_test 1.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf' } {7 70 98} @@ -282,13 +283,16 @@ if {$tn==3} breakpoint do_select_test 1.10 { SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld' } {10 13 17 31 35 51 58 88 89 90 93 100} - do_select_test 1.11 { - SELECT rowid FROM t1 - WHERE t1 MATCH '( - zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR - zk OR zkhdvkw OR zm OR zsmhnf - ) vgsld' - } {10 13 17 31 35 51 58 88 89 90 93 100} + + if { $fts3_simple_deferred_tokens_only==0 } { + do_select_test 1.11 { + SELECT rowid FROM t1 + WHERE t1 MATCH '( + zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR + zk OR zkhdvkw OR zm OR zsmhnf + ) vgsld' + } {10 13 17 31 35 51 58 88 89 90 93 100} + } do_select_test 2.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"' @@ -364,6 +368,7 @@ if {$tn==3} breakpoint foreach DO_MALLOC_TEST $dmt_modes { # Phrase search. + # do_select_test 5.$DO_MALLOC_TEST.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"' } {8 15 36 64 67 72} @@ -416,9 +421,11 @@ if {$tn==3} breakpoint do_select_test 6.2.2 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"' } {15 26 92 96} - do_select_test 6.2.3 { - SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' - } {8 15 26 92 96} + if {$fts3_simple_deferred_tokens_only==0} { + do_select_test 6.2.3 { + SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' + } {8 15 26 92 96} + } } set testprefix fts3defer diff --git a/test/permutations.test b/test/permutations.test index 9c48d9aa2a..4640ed1139 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -179,6 +179,7 @@ test_suite "fts3" -prefix "" -description { fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test + fts3sort.test fts3fault.test fts3malloc.test fts3matchinfo.test From be07ec582e358da8a7864d882a20596d6cb69dda Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Jun 2011 12:15:26 +0000 Subject: [PATCH 20/69] Fix an #ifdef of SQLITE_OMIT_VIRTUALTABLE that had an extra "_" character. FossilOrigin-Name: 93e0be2bbf16d66d97ea7344187139d254b11cc3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f04e836195..62c45edfca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sin\sthe\sWAL-mode\slogic\sfor\sread-only\sshared\smemory. -D 2011-06-02T17:24:49.997 +C Fix\san\s#ifdef\sof\sSQLITE_OMIT_VIRTUALTABLE\sthat\shad\san\sextra\s"_"\scharacter. +D 2011-06-03T12:15:26.459 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -236,7 +236,7 @@ F src/update.c 5bcb56e5c7380a2eecb0e71891dbd4ad7437748f F src/utf.c d83650c3ea08f7407bd9d0839d9885241c209c60 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 2ec37637fa70ab0c694d8327ee5dcedbc0621524 +F src/vdbe.c edfa3827d7a6fac2425bc10c0eb6e54342d2fa56 F src/vdbe.h d9c6123384189dc37d27beac1bf44688aa75b6cb F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 F src/vdbeapi.c 0eeadc75e44a30efd996d6af6e7c5a2488e35be8 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P efb20b9da6c7cb310a449cc818eaccd3d5bb4ab3 -R 7af7e431287e95dc91be1106acfd740b +P a13cfe616284f4ee86f0406e7b8fe8f9ba6e6990 +R 560ca19206a2de258e4ac41eb28208c5 U drh -Z e77c10de1c037b59f0516366918c7ba0 +Z 105c7f13fbfb0b81256f08e0207c0eef diff --git a/manifest.uuid b/manifest.uuid index b636e3ed7d..8cec759819 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a13cfe616284f4ee86f0406e7b8fe8f9ba6e6990 \ No newline at end of file +93e0be2bbf16d66d97ea7344187139d254b11cc3 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 439fe72540..97a0a6a555 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2585,7 +2585,7 @@ case OP_Savepoint: { }else{ nName = sqlite3Strlen30(zName); -#ifndef SQLITE_OMIT_VIRTUAL_TABLE +#ifndef SQLITE_OMIT_VIRTUALTABLE /* This call is Ok even if this savepoint is actually a transaction ** savepoint (and therefore should not prompt xSavepoint()) callbacks. ** If this is a transaction savepoint being opened, it is guaranteed From c7dc9bf88ed33e1676dd8e53b57eb330f8bf78fa Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Jun 2011 13:02:57 +0000 Subject: [PATCH 21/69] Fix the build when using SQLITE_OMIT_PRAGMA. FossilOrigin-Name: 051f4635bf1e9618b108c4177b8ecc5762ed08e7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 62c45edfca..e99630aa85 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\s#ifdef\sof\sSQLITE_OMIT_VIRTUALTABLE\sthat\shad\san\sextra\s"_"\scharacter. -D 2011-06-03T12:15:26.459 +C Fix\sthe\sbuild\swhen\susing\sSQLITE_OMIT_PRAGMA. +D 2011-06-03T13:02:57.441 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -171,7 +171,7 @@ F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c d548e31beafa792d1994b663a29a5303569efc4e -F src/pragma.c 9e778decc3ee9bcaf88904b4a3b0a4360aaf0eab +F src/pragma.c ebcd20f1e654f5cb3aeef864ed69c4697719fbaa F src/prepare.c e64261559a3187698a3e7e6c8b001a4f4f98dab4 F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a13cfe616284f4ee86f0406e7b8fe8f9ba6e6990 -R 560ca19206a2de258e4ac41eb28208c5 +P 93e0be2bbf16d66d97ea7344187139d254b11cc3 +R 4ed98f1ed7ddc5d80458937702f664db U drh -Z 105c7f13fbfb0b81256f08e0207c0eef +Z 234b0b60ed4ca4872a72109fff862187 diff --git a/manifest.uuid b/manifest.uuid index 8cec759819..ae169702be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -93e0be2bbf16d66d97ea7344187139d254b11cc3 \ No newline at end of file +051f4635bf1e9618b108c4177b8ecc5762ed08e7 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 799805c40b..cd2aab223c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -13,10 +13,6 @@ */ #include "sqliteInt.h" -/* Ignore this whole file if pragmas are disabled -*/ -#if !defined(SQLITE_OMIT_PRAGMA) - /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or @@ -53,6 +49,12 @@ u8 sqlite3GetBoolean(const char *z){ return getSafetyLevel(z)&1; } +/* The sqlite3GetBoolean() function is used by other modules but the +** remainder of this file is specific to PRAGMA processing. So omit +** the rest of the file if PRAGMAs are omitted from the build. +*/ +#if !defined(SQLITE_OMIT_PRAGMA) + /* ** Interpret the given string as a locking mode value. */ From 3043ac70c44024b0f0cab143392805dcf02e095f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Jun 2011 13:06:50 +0000 Subject: [PATCH 22/69] Add shell scripts used for testing compiler warnings (tool/warnings.sh), for verifying that the library exports the correct symbols (tool/symbols.sh), and to demonstrate building a full-featured command-line shell (tool/build-shell.sh). FossilOrigin-Name: 3aca9a92c8b29bb43f65f93593ba4defd65139dc --- manifest | 13 ++++++++----- manifest.uuid | 2 +- tool/build-shell.sh | 21 +++++++++++++++++++++ tool/symbols.sh | 34 ++++++++++++++++++++++++++++++++++ tool/warnings.sh | 14 ++++++++++++++ 5 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 tool/build-shell.sh create mode 100644 tool/symbols.sh create mode 100644 tool/warnings.sh diff --git a/manifest b/manifest index e99630aa85..c7ac33de9e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sbuild\swhen\susing\sSQLITE_OMIT_PRAGMA. -D 2011-06-03T13:02:57.441 +C Add\sshell\sscripts\sused\sfor\stesting\scompiler\swarnings\s(tool/warnings.sh),\nfor\sverifying\sthat\sthe\slibrary\sexports\sthe\scorrect\ssymbols\s(tool/symbols.sh),\nand\sto\sdemonstrate\sbuilding\sa\sfull-featured\scommand-line\sshell\n(tool/build-shell.sh). +D 2011-06-03T13:06:50.043 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -904,6 +904,7 @@ F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 +F tool/build-shell.sh 12aa4391073a777fcb6dcc490b219a018ae98bac F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 @@ -938,8 +939,10 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c +F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 93e0be2bbf16d66d97ea7344187139d254b11cc3 -R 4ed98f1ed7ddc5d80458937702f664db +F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d +P 051f4635bf1e9618b108c4177b8ecc5762ed08e7 +R 9da6c706bda9f5ba334ec4313d0142ea U drh -Z 234b0b60ed4ca4872a72109fff862187 +Z a1ce6be566aa33f6b6cc12b210f2a7c9 diff --git a/manifest.uuid b/manifest.uuid index ae169702be..face1e362d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -051f4635bf1e9618b108c4177b8ecc5762ed08e7 \ No newline at end of file +3aca9a92c8b29bb43f65f93593ba4defd65139dc \ No newline at end of file diff --git a/tool/build-shell.sh b/tool/build-shell.sh new file mode 100644 index 0000000000..54e8308095 --- /dev/null +++ b/tool/build-shell.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# +# This script demonstrates how to do a full-featured build of the sqlite3 +# command-line shell on Linux. +# +# SQLite source code should be in a sibling directory named "sqlite". For +# example, put SQLite sources in ~/sqlite/sqlite and run this script from +# ~/sqlite/bld. There should be an appropriate Makefile in the current +# directory as well. +# +make sqlite3.c +gcc -o sqlite3 -g -Os -I. \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_ENABLE_VFSTRACE \ + -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_FTS3 \ + -DSQLITE_ENABLE_RTREE \ + -DHAVE_READLINE \ + -DHAVE_USLEEP=1 \ + ../sqlite/src/shell.c ../sqlite/src/test_vfstrace.c \ + sqlite3.c -ldl -lreadline -lncurses diff --git a/tool/symbols.sh b/tool/symbols.sh new file mode 100644 index 0000000000..8aec00569a --- /dev/null +++ b/tool/symbols.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# +# Run this script in a directory that contains a valid SQLite makefile in +# order to verify that unintentionally exported symbols. +# +make sqlite3.c + +echo '****** Exported symbols from a build including RTREE, FTS4 & ICU ******' +gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + -DSQLITE_ENABLE_ICU \ + sqlite3.c +nm sqlite3.o | grep ' T ' | sort -k 3 + +echo '****** Surplus symbols from a build including RTREE, FTS4 & ICU ******' +nm sqlite3.o | grep ' T ' | grep -v ' sqlite3_' + +echo '****** Dependencies of the core. No extensions. No OS interface *******' +gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + -DSQLITE_OS_OTHER -DSQLITE_THREADSAFE=0 \ + sqlite3.c +nm sqlite3.o | grep ' U ' | sort -k 3 + +echo '****** Dependencies including RTREE & FTS4 *******' +gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + sqlite3.c +nm sqlite3.o | grep ' U ' | sort -k 3 diff --git a/tool/warnings.sh b/tool/warnings.sh new file mode 100644 index 0000000000..e1fa2b2d91 --- /dev/null +++ b/tool/warnings.sh @@ -0,0 +1,14 @@ +#/bin/sh +# +# Run this script in a directory with a working makefile to check for +# compiler warnings in SQLite. +# +make sqlite3.c +echo '********** No optimizations. Includes FTS4 and RTREE *********' +gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ + -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ + sqlite3.c +echo '********** Optimized -O3. Includes FTS4 and RTREE *********' +gcc -O3 -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ + -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ + sqlite3.c From 9fd301bb6ac5e9d7e017fff079d9034716ae1a09 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Jun 2011 13:28:22 +0000 Subject: [PATCH 23/69] Include more detailed version information in the command-line shell output. FossilOrigin-Name: 049c3c42fdefea8de7ec7008871963e37ce2d7bc --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 11 ++++++++--- tool/shell1.test | 5 +++-- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index c7ac33de9e..bcc31ebfb3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sshell\sscripts\sused\sfor\stesting\scompiler\swarnings\s(tool/warnings.sh),\nfor\sverifying\sthat\sthe\slibrary\sexports\sthe\scorrect\ssymbols\s(tool/symbols.sh),\nand\sto\sdemonstrate\sbuilding\sa\sfull-featured\scommand-line\sshell\n(tool/build-shell.sh). -D 2011-06-03T13:06:50.043 +C Include\smore\sdetailed\sversion\sinformation\sin\sthe\scommand-line\sshell\soutput. +D 2011-06-03T13:28:22.119 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,7 +178,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff -F src/shell.c decd04236a7ef26be5ef46d4ea963044bfad9a48 +F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 F src/sqlite.h.in 2f51e4f58b2b4626fcbd9938580e730cb5fb4985 F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 F src/sqliteInt.h 6e58c558c57c8f44011736d5fa5295eb3130f9de @@ -922,7 +922,7 @@ F tool/omittest.tcl b1dd290c1596e0f31fd335160a74ec5dfea3df4a F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/shell1.test 5542ecdc952f91121a835ed817e6feaf8988b333 +F tool/shell1.test 20dfe7099cf2afe37aecd69afb7678d14f7a0abf F tool/shell2.test 5dc76b8005b465f420fed8241621da7513060ff3 F tool/shell3.test 4fad469e8003938426355afdf34155f08c587836 F tool/shell4.test 35f9c3d452b4e76d5013c63e1fd07478a62f14ce @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 051f4635bf1e9618b108c4177b8ecc5762ed08e7 -R 9da6c706bda9f5ba334ec4313d0142ea +P 3aca9a92c8b29bb43f65f93593ba4defd65139dc +R 1eb21a9d7ed9685f625d6c1573544e8b U drh -Z a1ce6be566aa33f6b6cc12b210f2a7c9 +Z 70d86fceaffca6cead41b36e31773fcc diff --git a/manifest.uuid b/manifest.uuid index face1e362d..13a02e0d29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3aca9a92c8b29bb43f65f93593ba4defd65139dc \ No newline at end of file +049c3c42fdefea8de7ec7008871963e37ce2d7bc \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 8e9870a5a8..a54c922e87 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2302,6 +2302,11 @@ static int do_meta_command(char *zLine, struct callback_data *p){ enableTimer = booleanValue(azArg[1]); }else + if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ + printf("SQLite %s %s\n", + sqlite3_libversion(), sqlite3_sourceid()); + }else + if( c=='w' && strncmp(azArg[0], "width", n)==0 && nArg>1 ){ int j; assert( nArg<=ArraySize(azArg) ); @@ -2836,7 +2841,7 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( strcmp(z,"-version")==0 ){ - printf("%s\n", sqlite3_libversion()); + printf("%s %s\n", sqlite3_libversion(), sqlite3_sourceid()); return 0; }else if( strcmp(z,"-interactive")==0 ){ stdin_is_interactive = 1; @@ -2881,10 +2886,10 @@ int main(int argc, char **argv){ char *zHistory = 0; int nHistory; printf( - "SQLite version %s\n" + "SQLite version %s %.19s\n" "Enter \".help\" for instructions\n" "Enter SQL statements terminated with a \";\"\n", - sqlite3_libversion() + sqlite3_libversion(), sqlite3_sourceid() ); zHome = find_home_dir(); if( zHome ){ diff --git a/tool/shell1.test b/tool/shell1.test index 068f8ea25c..9dd9df5555 100644 --- a/tool/shell1.test +++ b/tool/shell1.test @@ -199,8 +199,9 @@ do_test shell1-1.15.3 { # -version show SQLite version do_test shell1-1.16.1 { - catchcmd "-version test.db" "" -} {0 3.7.7} + set x [catchcmd "-version test.db" ""] + regexp {0 \{3.\d.\d+ 20\d\d-[01]\d-\d\d \d\d:\d\d:\d\d [0-9a-f]+\}} $x +} 1 #---------------------------------------------------------------------------- # Test cases shell1-2.*: Basic "dot" command token parsing. From 45caededba6bd24dcbd31c5db1bb0d423fb8be9e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Jun 2011 14:19:10 +0000 Subject: [PATCH 24/69] Hush some harmless compiler warnings in the URI parsing logic. FossilOrigin-Name: 0206bc6f87bb9393218a380fc5b18039d334a8d8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index bcc31ebfb3..1099d5534a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Include\smore\sdetailed\sversion\sinformation\sin\sthe\scommand-line\sshell\soutput. -D 2011-06-03T13:28:22.119 +C Hush\ssome\sharmless\scompiler\swarnings\sin\sthe\sURI\sparsing\slogic. +D 2011-06-03T14:19:10.707 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -144,7 +144,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 3ae0d52da013a6326310655be6473fd472347b85 -F src/main.c 059daeed5876b3604f0192f838faf5f4db138901 +F src/main.c d67ea48933e4d63abba00578224a3319e6fbcbe1 F src/malloc.c 591aedb20ae40813f1045f2ef253438a334775d9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 3aca9a92c8b29bb43f65f93593ba4defd65139dc -R 1eb21a9d7ed9685f625d6c1573544e8b +P 049c3c42fdefea8de7ec7008871963e37ce2d7bc +R 06582f4582a8157c228d4b288a346794 U drh -Z 70d86fceaffca6cead41b36e31773fcc +Z 0eb02a6eb0f61a953a539b049b2322b5 diff --git a/manifest.uuid b/manifest.uuid index 13a02e0d29..a5c8a3d8b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -049c3c42fdefea8de7ec7008871963e37ce2d7bc \ No newline at end of file +0206bc6f87bb9393218a380fc5b18039d334a8d8 \ No newline at end of file diff --git a/src/main.c b/src/main.c index c99d396546..d3386eb0cd 100644 --- a/src/main.c +++ b/src/main.c @@ -1938,9 +1938,9 @@ int sqlite3ParseUri( const char *z; int mode; } *aMode = 0; - char *zModeType; - int mask; - int limit; + char *zModeType = 0; + int mask = 0; + int limit = 0; if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ static struct OpenMode aCacheMode[] = { From 68f2a57698780de49312cdb0f11732c521955d60 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Jun 2011 17:50:49 +0000 Subject: [PATCH 25/69] Factor an "if" out of a loop in balance_nonroot() for about a 1% performance increase. FossilOrigin-Name: 1bd72d0c616e20fdb395c72ecd96579090ae26cb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 26 ++++++++++++++++++++------ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 1099d5534a..b636720757 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Hush\ssome\sharmless\scompiler\swarnings\sin\sthe\sURI\sparsing\slogic. -D 2011-06-03T14:19:10.707 +C Factor\san\s"if"\sout\sof\sa\sloop\sin\sbalance_nonroot()\sfor\sabout\sa\s1%\sperformance\nincrease. +D 2011-06-03T17:50:49.782 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,7 +122,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 0d3b39dcb79565c053e35fc12713f12d8a74d6a9 +F src/btree.c 12aa3b71359c888984223cb2bcf691cf2d7753ae F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c c10ab9e2c77ade99dee23554787f8acfc0c231fc @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 049c3c42fdefea8de7ec7008871963e37ce2d7bc -R 06582f4582a8157c228d4b288a346794 +P 0206bc6f87bb9393218a380fc5b18039d334a8d8 +R 9437f6a8af920a11b9b18fbeccb53c6b U drh -Z 0eb02a6eb0f61a953a539b049b2322b5 +Z d6270d9f0f6d36a33c5a8dae47c51fc0 diff --git a/manifest.uuid b/manifest.uuid index a5c8a3d8b3..a78d2a2cc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0206bc6f87bb9393218a380fc5b18039d334a8d8 \ No newline at end of file +1bd72d0c616e20fdb395c72ecd96579090ae26cb \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 972a2f608c..9e9677cc42 100644 --- a/src/btree.c +++ b/src/btree.c @@ -857,6 +857,8 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ */ #define findCell(P,I) \ ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)]))) +#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) + /* ** This a more complex version of findCell() that works for @@ -6016,12 +6018,24 @@ static int balance_nonroot( memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); limit = pOld->nCell+pOld->nOverflow; - for(j=0; jnOverflow>0 ){ + for(j=0; jaData; + u16 maskPage = pOld->maskPage; + u16 cellOffset = pOld->cellOffset; + for(j=0; j Date: Fri, 3 Jun 2011 18:00:19 +0000 Subject: [PATCH 26/69] FTS changes: Remove unreachable code. Fix bugs. When processing a large doclist incrementally, read from disk incrementally too. FossilOrigin-Name: a4c7e2820824e82580730c36f85aede2efa66754 --- ext/fts3/fts3.c | 991 ++++++++-------------------------------- ext/fts3/fts3Int.h | 25 +- ext/fts3/fts3_expr.c | 5 +- ext/fts3/fts3_snippet.c | 63 +-- ext/fts3/fts3_write.c | 174 +++++-- manifest | 20 +- manifest.uuid | 2 +- 7 files changed, 345 insertions(+), 935 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index a7f176a700..90b081b8c5 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -313,9 +313,6 @@ SQLITE_EXTENSION_INIT1 #endif -static char *fts3EvalPhrasePoslist(Fts3Phrase *, int *); -static sqlite3_int64 fts3EvalPhraseDocid(Fts3Phrase *); - /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. @@ -1210,9 +1207,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ */ if( pInfo->nOrderBy==1 ){ struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; - if( pOrder->desc==0 - && (pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1) - ){ + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ if( pOrder->desc ){ pInfo->idxStr = "DESC"; }else{ @@ -1261,8 +1256,6 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } -static int fts3RowidMethod(sqlite3_vtab_cursor *, sqlite3_int64*); - /* ** Position the pCsr->pStmt statement so that it is on the row ** of the %_content table that contains the last match. Return @@ -1452,7 +1445,7 @@ static int fts3SelectLeaf( int nBlob; /* Size of zBlob in bytes */ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); } @@ -1462,7 +1455,7 @@ static int fts3SelectLeaf( } if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); } if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); @@ -2194,35 +2187,6 @@ static int fts3TermSelectCb( return SQLITE_OK; } -static int fts3DeferredTermSelect( - Fts3DeferredToken *pToken, /* Phrase token */ - int isTermPos, /* True to include positions */ - int *pnOut, /* OUT: Size of list */ - char **ppOut /* OUT: Body of list */ -){ - char *aSource; - int nSource; - - aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); - if( !aSource ){ - *pnOut = 0; - *ppOut = 0; - }else if( isTermPos ){ - *ppOut = sqlite3_malloc(nSource); - if( !*ppOut ) return SQLITE_NOMEM; - memcpy(*ppOut, aSource, nSource); - *pnOut = nSource; - }else{ - sqlite3_int64 docid; - *pnOut = sqlite3Fts3GetVarint(aSource, &docid); - *ppOut = sqlite3_malloc(*pnOut); - if( !*ppOut ) return SQLITE_NOMEM; - sqlite3Fts3PutVarint(*ppOut, docid); - } - - return SQLITE_OK; -} - /* ** Append SegReader object pNew to the end of the pCsr->apSegment[] array. */ @@ -2512,198 +2476,6 @@ static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ return nDoc; } -/* -** Call sqlite3Fts3DeferToken() for each token in the expression pExpr. -*/ -static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc = SQLITE_OK; - if( pExpr ){ - rc = fts3DeferExpression(pCsr, pExpr->pLeft); - if( rc==SQLITE_OK ){ - rc = fts3DeferExpression(pCsr, pExpr->pRight); - } - if( pExpr->eType==FTSQUERY_PHRASE ){ - int iCol = pExpr->pPhrase->iColumn; - int i; - for(i=0; rc==SQLITE_OK && ipPhrase->nToken; i++){ - Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; - if( pToken->pDeferred==0 ){ - rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); - } - } - } - } - return rc; -} - -/* -** This function removes the position information from a doclist. When -** called, buffer aList (size *pnList bytes) contains a doclist that includes -** position information. This function removes the position information so -** that aList contains only docids, and adjusts *pnList to reflect the new -** (possibly reduced) size of the doclist. -*/ -static void fts3DoclistStripPositions( - char *aList, /* IN/OUT: Buffer containing doclist */ - int *pnList /* IN/OUT: Size of doclist in bytes */ -){ - if( aList ){ - char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Input cursor */ - char *pOut = aList; /* Output cursor */ - - while( piColumn; - int isTermPos = (pPhrase->nToken>1 || isReqPos); - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - int isFirst = 1; - - int iPrevTok = 0; - int nDoc = 0; - - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok; /* Token to find doclist for */ - int iTok = 0; /* The token being queried this iteration */ - char *pList = 0; /* Pointer to token doclist */ - int nList = 0; /* Size of buffer at pList */ - - /* Select a token to process. If this is an xFilter() call, then tokens - ** are processed in order from least to most costly. Otherwise, tokens - ** are processed in the order in which they occur in the phrase. - */ - if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ - assert( isReqPos ); - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - if( pTok->bFulltext==0 ) continue; - }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - }else{ - int nMinCost = 0x7FFFFFFF; - int jj; - - /* Find the remaining token with the lowest cost. */ - for(jj=0; jjnToken; jj++){ - Fts3MultiSegReader *pSegcsr = pPhrase->aToken[jj].pSegcsr; - if( pSegcsr && pSegcsr->nCostnCost; - } - } - pTok = &pPhrase->aToken[iTok]; - - /* This branch is taken if it is determined that loading the doclist - ** for the next token would require more IO than loading all documents - ** currently identified by doclist pOut/nOut. No further doclists will - ** be loaded from the full-text index for this phrase. - */ - if( nMinCost>nDoc && ii>0 ){ - rc = fts3DeferExpression(pCsr, pCsr->pExpr); - break; - } - } - - if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ - rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); - }else{ - if( pTok->pSegcsr ){ - rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); - } - pTok->bFulltext = 1; - } - assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 ); - if( rc!=SQLITE_OK ) break; - - if( isFirst ){ - pOut = pList; - nOut = nList; - if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ - nDoc = fts3DoclistCountDocids(1, pOut, nOut); - } - isFirst = 0; - iPrevTok = iTok; - }else{ - /* Merge the new term list and the current output. */ - char *aLeft, *aRight; - int nLeft, nRight; - int nDist; - int mt; - - /* If this is the final token of the phrase, and positions were not - ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. - ** This drops the position information from the output list. - */ - mt = MERGE_POS_PHRASE; - if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; - - assert( iPrevTok!=iTok ); - if( iPrevToknToken ){ - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); - fts3DoclistStripPositions(pOut, &nOut); - } - *paOut = pOut; - *pnOut = nOut; - }else{ - sqlite3_free(pOut); - } - return rc; -} - /* ** This function merges two doclists according to the requirements of a ** NEAR operator. @@ -2746,408 +2518,6 @@ static int fts3NearMerge( return rc; } -/* -** This function is used as part of the processing for the snippet() and -** offsets() functions. -** -** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both -** have their respective doclists (including position information) loaded -** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from -** each doclist that are not within nNear tokens of a corresponding entry -** in the other doclist. -*/ -int sqlite3Fts3ExprNearTrim(Fts3Expr *pELeft, Fts3Expr *pERight, int nNear){ - int rc; /* Return code */ - Fts3Phrase *pLeft = pELeft->pPhrase; - Fts3Phrase *pRight = pERight->pPhrase; - - assert( pELeft->eType==FTSQUERY_PHRASE && pLeft ); - assert( pERight->eType==FTSQUERY_PHRASE && pRight ); - assert( pLeft->isLoaded && pRight->isLoaded ); - - if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ - sqlite3_free(pLeft->aDoclist); - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = 0; - pLeft->aDoclist = 0; - rc = SQLITE_OK; - }else{ - char *aOut; /* Buffer in which to assemble new doclist */ - int nOut; /* Size of buffer aOut in bytes */ - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist, - pRight->nToken, pRight->aDoclist, pRight->nDoclist, - &aOut, &nOut - ); - if( rc!=SQLITE_OK ) return rc; - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = aOut; - pRight->nDoclist = nOut; - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pRight->nToken, pRight->aDoclist, pRight->nDoclist, - pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist, - &aOut, &nOut - ); - sqlite3_free(pLeft->aDoclist); - pLeft->aDoclist = aOut; - pLeft->nDoclist = nOut; - } - return rc; -} - - -/* -** Allocate an Fts3SegReaderArray for each token in the expression pExpr. -** The allocated objects are stored in the Fts3PhraseToken.pArray member -** variables of each token structure. -*/ -static int fts3ExprAllocateSegReaders( - Fts3Cursor *pCsr, /* FTS3 table */ - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - int *pnExpr /* OUT: Number of AND'd expressions */ -){ - int rc = SQLITE_OK; /* Return code */ - - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); - if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ - (*pnExpr)++; - pnExpr = 0; - } - - if( pExpr->eType==FTSQUERY_PHRASE ){ - int ii; /* Used to iterate through phrase tokens */ - Fts3Phrase *pPhrase = pExpr->pPhrase; - - for(ii=0; rc==SQLITE_OK && iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pSegcsr==0 ){ - rc = sqlite3Fts3TermSegReaderCursor( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr - ); - } - } - }else{ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); - if( rc==SQLITE_OK ){ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); - } - } - return rc; -} - -/* -** Free the Fts3SegReaderArray objects associated with each token in the -** expression pExpr. In other words, this function frees the resources -** allocated by fts3ExprAllocateSegReaders(). -*/ -static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - if( pPhrase ){ - int kk; - for(kk=0; kknToken; kk++){ - fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr); - pPhrase->aToken[kk].pSegcsr = 0; - } - } - fts3ExprFreeSegReaders(pExpr->pLeft); - fts3ExprFreeSegReaders(pExpr->pRight); - } -} - -/* -** Return the sum of the costs of all tokens in the expression pExpr. This -** function must be called after Fts3SegReaderArrays have been allocated -** for all tokens using fts3ExprAllocateSegReaders(). -*/ -static int fts3ExprCost(Fts3Expr *pExpr){ - int nCost; /* Return value */ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - nCost = 0; - for(ii=0; iinToken; ii++){ - Fts3MultiSegReader *pSegcsr = pPhrase->aToken[ii].pSegcsr; - if( pSegcsr ) nCost += pSegcsr->nCost; - } - }else{ - nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); - } - return nCost; -} - -/* -** The following is a helper function (and type) for fts3EvalExpr(). It -** must be called after Fts3SegReaders have been allocated for every token -** in the expression. See the context it is called from in fts3EvalExpr() -** for further explanation. -*/ -typedef struct ExprAndCost ExprAndCost; -struct ExprAndCost { - Fts3Expr *pExpr; - int nCost; -}; -static void fts3ExprAssignCosts( - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ -){ - if( pExpr->eType==FTSQUERY_AND ){ - fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); - fts3ExprAssignCosts(pExpr->pRight, ppExprCost); - }else{ - (*ppExprCost)->pExpr = pExpr; - (*ppExprCost)->nCost = fts3ExprCost(pExpr); - (*ppExprCost)++; - } -} - -/* -** Evaluate the full-text expression pExpr against FTS3 table pTab. Store -** the resulting doclist in *paOut and *pnOut. This routine mallocs for -** the space needed to store the output. The caller is responsible for -** freeing the space when it has finished. -** -** This function is called in two distinct contexts: -** -** * From within the virtual table xFilter() method. In this case, the -** output doclist contains entries for all rows in the table, based on -** data read from the full-text index. -** -** In this case, if the query expression contains one or more tokens that -** are very common, then the returned doclist may contain a superset of -** the documents that actually match the expression. -** -** * From within the virtual table xNext() method. This call is only made -** if the call from within xFilter() found that there were very common -** tokens in the query expression and did return a superset of the -** matching documents. In this case the returned doclist contains only -** entries that correspond to the current row of the table. Instead of -** reading the data for each token from the full-text index, the data is -** already available in-memory in the Fts3PhraseToken.pDeferred structures. -** See fts3EvalDeferred() for how it gets there. -** -** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is -** required) Fts3Cursor.doDeferred==1. -** -** If the SQLite invokes the snippet(), offsets() or matchinfo() function -** as part of a SELECT on an FTS3 table, this function is called on each -** individual phrase expression in the query. If there were very common tokens -** found in the xFilter() call, then this function is called once for phrase -** for each row visited, and the returned doclist contains entries for the -** current row only. Otherwise, if there were no very common tokens, then this -** function is called once only for each phrase in the query and the returned -** doclist contains entries for all rows of the table. -** -** Fts3Cursor.doDeferred==1 when this function is called on phrases as a -** result of a snippet(), offsets() or matchinfo() invocation. -*/ -static int fts3EvalExpr( - Fts3Cursor *p, /* Virtual table cursor handle */ - Fts3Expr *pExpr, /* Parsed fts3 expression */ - char **paOut, /* OUT: Pointer to malloc'd result buffer */ - int *pnOut, /* OUT: Size of buffer at *paOut */ - int isReqPos /* Require positions in output buffer */ -){ - int rc = SQLITE_OK; /* Return code */ - - /* Zero the output parameters. */ - *paOut = 0; - *pnOut = 0; - - if( pExpr ){ - assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT - || pExpr->eType==FTSQUERY_PHRASE - ); - assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); - - if( pExpr->eType==FTSQUERY_PHRASE ){ - rc = fts3PhraseSelect(p, pExpr->pPhrase, - isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), - paOut, pnOut - ); - fts3ExprFreeSegReaders(pExpr); - }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ - ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ - int nExpr = 0; /* Size of aExpr[] */ - char *aRet = 0; /* Doclist to return to caller */ - int nRet = 0; /* Length of aRet[] in bytes */ - int nDoc = 0x7FFFFFFF; - - assert( !isReqPos ); - - rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); - if( rc==SQLITE_OK ){ - assert( nExpr>1 ); - aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); - if( !aExpr ) rc = SQLITE_NOMEM; - } - if( rc==SQLITE_OK ){ - int ii; /* Used to iterate through expressions */ - - fts3ExprAssignCosts(pExpr, &aExpr); - aExpr -= nExpr; - for(ii=0; iipExpr && (pBest==0 || pCand->nCostnCost) ){ - pBest = pCand; - } - } - - if( pBest->nCost>nDoc ){ - rc = fts3DeferExpression(p, p->pExpr); - break; - }else{ - rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); - if( rc!=SQLITE_OK ) break; - pBest->pExpr = 0; - if( ii==0 ){ - aRet = aNew; - nRet = nNew; - nDoc = fts3DoclistCountDocids(0, aRet, nRet); - }else{ - fts3DoclistMerge( - MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc - ); - sqlite3_free(aNew); - } - } - } - } - - if( rc==SQLITE_OK ){ - *paOut = aRet; - *pnOut = nRet; - }else{ - assert( *paOut==0 ); - sqlite3_free(aRet); - } - sqlite3_free(aExpr); - fts3ExprFreeSegReaders(pExpr); - - }else{ - char *aLeft; - char *aRight; - int nLeft; - int nRight; - - assert( pExpr->eType==FTSQUERY_NEAR - || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_NOT - || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) - ); - - if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) - && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) - ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: { - Fts3Expr *pLeft; - Fts3Expr *pRight; - int mergetype = MERGE_NEAR; - if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ - mergetype = MERGE_POS_NEAR; - } - pLeft = pExpr->pLeft; - while( pLeft->eType==FTSQUERY_NEAR ){ - pLeft=pLeft->pRight; - } - pRight = pExpr->pRight; - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->eType==FTSQUERY_PHRASE ); - - rc = fts3NearMerge(mergetype, pExpr->nNear, - pLeft->pPhrase->nToken, aLeft, nLeft, - pRight->pPhrase->nToken, aRight, nRight, - paOut, pnOut - ); - sqlite3_free(aLeft); - break; - } - - case FTSQUERY_OR: { - /* Allocate a buffer for the output. The maximum size is the - ** sum of the sizes of the two input buffers. The +1 term is - ** so that a buffer of zero bytes is never allocated - this can - ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. - */ - char *aBuffer = sqlite3_malloc(nRight+nLeft+1); - rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aBuffer; - sqlite3_free(aLeft); - break; - } - - default: { - assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); - fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aLeft; - break; - } - } - } - sqlite3_free(aRight); - } - } - - assert( rc==SQLITE_OK || *paOut==0 ); - return rc; -} - -/* -** This function is called from within xNext() for each row visited by -** an FTS3 query. If evaluating the FTS3 query expression within xFilter() -** was able to determine the exact set of matching rows, this function sets -** *pbRes to true and returns SQLITE_IO immediately. -** -** Otherwise, if evaluating the query expression within xFilter() returned a -** superset of the matching documents instead of an exact set (this happens -** when the query includes very common tokens and it is deemed too expensive to -** load their doclists from disk), this function tests if the current row -** really does match the FTS3 query. -** -** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK -** is returned and *pbRes is set to true if the current row matches the -** FTS3 query (and should be included in the results returned to SQLite), or -** false otherwise. -*/ -static int fts3EvalDeferred( - Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ - int *pbRes /* OUT: Set to true if row is a match */ -){ - int rc = SQLITE_OK; - if( pCsr->pDeferred==0 ){ - *pbRes = 1; - }else{ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - sqlite3Fts3FreeDeferredDoclists(pCsr); - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); - } - if( rc==SQLITE_OK ){ - char *a = 0; - int n = 0; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); - assert( n>=0 ); - *pbRes = (n>0); - sqlite3_free(a); - } - } - return rc; -} - /* ** Advance the cursor to the next row in the %_content table that ** matches the search criteria. For a MATCH search, this will be @@ -3160,43 +2530,19 @@ static int fts3EvalDeferred( ** subsequently to determine whether or not an EOF was hit. */ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ - int res; - int rc = SQLITE_OK; /* Return code */ + int rc; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - if( pCsr->bIncremental ){ - rc = sqlite3Fts3EvalNext(pCsr, pCsr->pExpr); + if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + }else{ + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; + } }else{ - pCsr->eEvalmode = FTS3_EVAL_NEXT; - do { - if( pCsr->aDoclist==0 ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - break; - } - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); - }else{ - if( pCsr->desc==0 ){ - if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ - pCsr->isEof = 1; - break; - } - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); - }else{ - fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId); - if( pCsr->pNextId<=pCsr->aDoclist ){ - pCsr->isEof = 1; - break; - } - } - sqlite3_reset(pCsr->pStmt); - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - } - }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); + rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); } - return rc; } @@ -3242,6 +2588,9 @@ static int fts3FilterMethod( sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + pCsr->bDesc = (idxStr && idxStr[0]=='D'); + pCsr->eSearch = (i16)idxNum; + if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); @@ -3264,7 +2613,6 @@ static int fts3FilterMethod( rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - pCsr->bIncremental = 1; rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); sqlite3Fts3SegmentsClose(p); @@ -3296,22 +2644,6 @@ static int fts3FilterMethod( if( rc!=SQLITE_OK ) return rc; } - assert( pCsr->desc==0 ); - pCsr->eSearch = (i16)idxNum; - if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){ - sqlite3_int64 iDocid = 0; - char *csr = pCsr->aDoclist; - while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){ - fts3GetDeltaVarint(&csr, &iDocid); - } - pCsr->pNextId = csr; - pCsr->iPrevId = iDocid; - pCsr->desc = 1; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->eEvalmode = FTS3_EVAL_NEXT; - return SQLITE_OK; - } return fts3NextMethod(pCursor); } @@ -3331,18 +2663,7 @@ static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - if( pCsr->bIncremental ){ - *pRowid = sqlite3Fts3EvalDocid(pCsr, pCsr->pExpr); - }else if( pCsr->aDoclist ){ - *pRowid = pCsr->iPrevId; - }else{ - /* This branch runs if the query is implemented using a full-table scan - ** (not using the full-text index). In this case grab the rowid from the - ** SELECT statement. - */ - assert( pCsr->isRequireSeek==0 ); - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); - } + *pRowid = pCsr->iPrevId; return SQLITE_OK; } @@ -3449,46 +2770,6 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ return SQLITE_OK; } -/* -** Load the doclist associated with expression pExpr to pExpr->aDoclist. -** The loaded doclist contains positions as well as the document ids. -** This is used by the matchinfo(), snippet() and offsets() auxillary -** functions. -*/ -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc = SQLITE_OK; - if( pCsr->bIncremental==0 ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->eType==FTSQUERY_PHRASE && pPhrase ); - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - rc = fts3EvalExpr(pCsr, pExpr, &pPhrase->aDoclist, &pPhrase->nDoclist, 1); - } - return rc; -} - -/* -** TODO: This is something to do with matchinfo(). Similar to -** sqlite3ExprLoadDoclists() but slightly different. -** -** UPDATE: Only used when there are deferred tokens. -*/ -int sqlite3Fts3ExprLoadFtDoclist( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - char **paDoclist, - int *pnDoclist -){ - int rc = SQLITE_OK; - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - assert( pCsr->bIncremental==0 ); - pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; - rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); - pCsr->eEvalmode = FTS3_EVAL_NEXT; - return rc; -} - - /* ** When called, *ppPoslist must point to the byte immediately following the ** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function @@ -3909,12 +3190,6 @@ int sqlite3_extension_init( } #endif -/************************************************************************* -************************************************************************** -************************************************************************** -************************************************************************** -*************************************************************************/ - /* ** Allocate an Fts3MultiSegReader for each token in the expression headed @@ -4021,6 +3296,8 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ int nPoslist = 0; int iPrev = -1; + assert( pPhrase->doclist.bFreeList==0 ); + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; Fts3DeferredToken *pDeferred = pToken->pDeferred; @@ -4070,6 +3347,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ pPhrase->doclist.pList = aPoslist; pPhrase->doclist.nList = nPoslist; pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; }else{ int nDistance; char *p1; @@ -4094,13 +3372,14 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ pPhrase->doclist.pList = aOut; if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); - sqlite3_free(aPoslist); }else{ sqlite3_free(aOut); pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; } + sqlite3_free(aPoslist); } } @@ -4113,16 +3392,20 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ ** ** fts3EvalPhraseStart() ** fts3EvalPhraseNext() -** fts3EvalPhraseReset() ** ** May be used with a phrase object after fts3EvalAllocateReaders() has been ** called to iterate through the set of docids that match the phrase. ** ** After a successful call to fts3EvalPhraseNext(), the following two ** functions may be called to access the current docid and position-list. -** -** fts3EvalPhraseDocid() -** fts3EvalPhrasePoslist() +*/ + + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ int rc; @@ -4131,7 +3414,7 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ assert( pList->aAll==0 ); - if( p->nToken==1 && bOptOk==1 + if( pCsr->bDesc==0 && bOptOk==1 && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup ){ /* Use the incremental approach. */ @@ -4142,7 +3425,6 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ p->bIncr = 1; }else{ - /* Load the full doclist for the phrase into memory. */ rc = fts3EvalPhraseLoad(pCsr, p); p->bIncr = 0; @@ -4161,22 +3443,60 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ ** 1 before returning. Otherwise, if no error occurs and the iterator is ** successfully advanced, *pbEof is set to 0. */ -static int fts3EvalPhraseNext(Fts3Cursor *pCsr, Fts3Phrase *p, u8 *pbEof){ +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, + Fts3Phrase *p, + u8 *pbEof +){ int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; if( p->bIncr ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; assert( p->nToken==1 ); rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, - &p->doclist.iDocid, &p->doclist.pList, &p->doclist.nList + &pDL->iDocid, &pDL->pList, &pDL->nList ); - if( rc==SQLITE_OK && !p->doclist.pList ){ + if( rc==SQLITE_OK && !pDL->pList ){ *pbEof = 1; } - }else{ - char *pIter; - Fts3Doclist *pDL = &p->doclist; + }else if( pCsr->bDesc && pDL->aAll ){ + if( pDL->pNextDocid==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext; + char *pDocid = pDL->aAll; + char *pEnd = &pDocid[pDL->nAll]; + + while( pDocidpNextDocid = pDocid; + pDL->pList = pDocid; + fts3PoslistCopy(0, &pDocid); + } + pDL->nList = (pEnd - pDL->pList); + pDL->iDocid = iDocid; + }else{ + + assert( *pbEof==0 ); + assert( pDL->pNextDocid>pDL->aAll ); + + fts3GetReverseDeltaVarint( + &pDL->pNextDocid, pDL->aAll, &pDL->iDocid + ); + if( pDL->pNextDocid==pDL->aAll ){ + *pbEof = 1; + }else{ + char *pSave = pDL->pNextDocid; + fts3ReversePoslist(pDL->aAll, &pDL->pNextDocid); + pDL->pList = pDL->pNextDocid; + pDL->nList = pSave - pDL->pNextDocid; + } + } + + }else{ + + char *pIter; if( pDL->pNextDocid ){ pIter = pDL->pNextDocid; }else{ @@ -4199,21 +3519,6 @@ static int fts3EvalPhraseNext(Fts3Cursor *pCsr, Fts3Phrase *p, u8 *pbEof){ return rc; } -static int fts3EvalPhraseReset(Fts3Cursor *pCsr, Fts3Phrase *p){ - return SQLITE_OK; -} - -static sqlite3_int64 fts3EvalPhraseDocid(Fts3Phrase *p){ - return p->doclist.iDocid; -} - -static char *fts3EvalPhrasePoslist(Fts3Phrase *p, int *pnList){ - if( pnList ){ - *pnList = p->doclist.nList; - } - return p->doclist.pList; -} - static void fts3EvalStartReaders( Fts3Cursor *pCsr, Fts3Expr *pExpr, @@ -4495,16 +3800,33 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ return rc; } +static void fts3EvalFreeDeferredDoclist(Fts3Phrase *pPhrase){ + if( pPhrase->doclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; + } +} + +/* +** This macro is used by the fts3EvalNext() function. The two arguments are +** 64-bit docid values. If the current query is "ORDER BY docid ASC", then +** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then +** it returns (i2 - i1). This allows the same code to be used for merging +** doclists in ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) + static void fts3EvalNext( Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc ){ if( *pRc==SQLITE_OK ){ - pExpr->bStart = 1; - switch( pExpr->eType ){ + switch( pExpr->eType ){ case FTSQUERY_NEAR: case FTSQUERY_AND: { Fts3Expr *pLeft = pExpr->pLeft; @@ -4524,7 +3846,7 @@ static void fts3EvalNext( fts3EvalNext(pCsr, pRight, pRc); while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - int iDiff = pLeft->iDocid - pRight->iDocid; + int iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); if( iDiff==0 ) break; if( iDiff<0 ){ fts3EvalNext(pCsr, pLeft, pRc); @@ -4542,23 +3864,23 @@ static void fts3EvalNext( case FTSQUERY_OR: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; + int iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - if( pLeft->iDocid==pRight->iDocid ){ + if( iCmp==0 ){ fts3EvalNext(pCsr, pLeft, pRc); fts3EvalNext(pCsr, pRight, pRc); - }else if( - pRight->bEof || (pLeft->bEof==0 && pLeft->iDocidiDocid) - ){ + }else if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ fts3EvalNext(pCsr, pLeft, pRc); }else{ fts3EvalNext(pCsr, pRight, pRc); } pExpr->bEof = (pLeft->bEof && pRight->bEof); - if( pRight->bEof || (pLeft->bEof==0 && pLeft->iDocidiDocid) ){ + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ pExpr->iDocid = pLeft->iDocid; }else{ pExpr->iDocid = pRight->iDocid; @@ -4578,7 +3900,10 @@ static void fts3EvalNext( do { fts3EvalNext(pCsr, pLeft, pRc); if( pLeft->bEof ) break; - while( !*pRc && !pRight->bEof && pRight->iDocidiDocid ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ fts3EvalNext(pCsr, pRight, pRc); } }while( !pRight->bEof && pRight->iDocid==pLeft->iDocid && !*pRc ); @@ -4587,11 +3912,13 @@ static void fts3EvalNext( break; } - default: - assert( pExpr->eType==FTSQUERY_PHRASE ); - *pRc = fts3EvalPhraseNext(pCsr, pExpr->pPhrase, &pExpr->bEof); - pExpr->iDocid = fts3EvalPhraseDocid(pExpr->pPhrase); + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalFreeDeferredDoclist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; break; + } } } } @@ -4622,12 +3949,14 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ ); break; - default: - assert( pExpr->eType==FTSQUERY_PHRASE ); - *pRc = fts3EvalDeferredPhrase(pCsr, pExpr->pPhrase); - bHit = (pExpr->pPhrase->doclist.pList!=0); + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalFreeDeferredDoclist(pPhrase); + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); pExpr->iDocid = pCsr->iPrevId; break; + } } } return bHit; @@ -4640,6 +3969,9 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ ** ** 2. After scanning the current FTS table row for the deferred tokens, ** it is determined that the row does not match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. */ static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; @@ -4658,18 +3990,21 @@ static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ } /* -** Advance to the next document that matches the expression passed as an -** argument. +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. */ -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr){ +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; assert( pCsr->isEof==0 ); - assert( pCsr->bIncremental ); if( pExpr==0 ){ pCsr->isEof = 1; }else{ do { - sqlite3_reset(pCsr->pStmt); + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); fts3EvalNext(pCsr, pExpr, &rc); pCsr->isEof = pExpr->bEof; pCsr->isRequireSeek = 1; @@ -4680,17 +4015,24 @@ int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr){ return rc; } -int sqlite3Fts3EvalFinish(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - return SQLITE_OK; -} - -sqlite3_int64 sqlite3Fts3EvalDocid(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - return pExpr->iDocid; -} - /* ** Return a pointer to the entire doclist, including positions, associated -** with the phrase passed as the second argument. +** with the phrase passed as the second argument. It is illegal to call +** this function if the phrase consists entirely of deferred tokens. +** +** TODO: This function is only used by the code for the matchinfo('x') +** auxiliary function - to obtain the following two values: +** +** 1. The total number of times the phrase appears in each column in all +** rows in the FTS table. +** +** 2. For each column, the total number of rows in the FTS table for which +** the phrase appears at least once in the column. +** +** It would be better if there was an sqlite3Fts3EvalXXX() function +** specifically to retrieve these values. If that were done, the concept +** of which tokens are deferred or incremental would be entirely encapsulated +** within the sqlite3Fts3EvalXXX()/fts3EvalXXX() functions in this file. */ int sqlite3Fts3EvalPhraseDoclist( Fts3Cursor *pCsr, /* FTS3 cursor object */ @@ -4701,6 +4043,12 @@ int sqlite3Fts3EvalPhraseDoclist( int rc = SQLITE_OK; Fts3Phrase *pPhrase = pExpr->pPhrase; + /* It is illegal to call this function if the phrase is entirely deferred + ** (it may contain some deferred tokens, but must also contain at least + ** one token for which the doclist may be read from the full-text index). + */ + assert( !pExpr->bDeferred ); + if( pPhrase->bIncr ){ /* This phrase was being loaded from disk incrementally. But the ** matchinfo() function requires that the entire doclist be loaded into @@ -4731,10 +4079,28 @@ int sqlite3Fts3EvalPhraseDoclist( return rc; } +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ char *sqlite3Fts3EvalPhrasePoslist( Fts3Cursor *pCsr, /* FTS3 cursor object */ Fts3Expr *pExpr, /* Phrase to return doclist for */ - sqlite3_int64 iDocid, /* Docid to return position list for */ int iCol /* Column to return position list for */ ){ Fts3Phrase *pPhrase = pExpr->pPhrase; @@ -4745,7 +4111,7 @@ char *sqlite3Fts3EvalPhrasePoslist( assert( iCol>=0 && iColnColumn ); if( !pIter || pExpr->bEof - || pExpr->iDocid!=iDocid + || pExpr->iDocid!=pCsr->iPrevId || (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ return 0; @@ -4768,13 +4134,20 @@ char *sqlite3Fts3EvalPhrasePoslist( return ((iCol==iThis)?pIter:0); } +/* +** Free all components of the Fts3Phrase structure that were allocated by +** the eval module. +*/ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ - int i; - sqlite3_free(pPhrase->doclist.aAll); - memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); - for(i=0; inToken; i++){ - fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); - pPhrase->aToken[i].pSegcsr = 0; + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalFreeDeferredDoclist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } } } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 0d69029dfb..c6834d37b6 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -230,14 +230,13 @@ struct Fts3Cursor { u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ - int bIncremental; /* True to use incremental querying */ int nPhrase; /* Number of matchable phrases in query */ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ - int desc; /* True to sort in descending order */ + int bDesc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ @@ -274,9 +273,10 @@ struct Fts3Cursor { struct Fts3Doclist { char *aAll; /* Array containing doclist (or NULL) */ int nAll; /* Size of a[] in bytes */ - - sqlite3_int64 iDocid; /* Current docid (if p!=0) */ char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ char *pList; /* Pointer to position list following iDocid */ int nList; /* Length of position list */ } doclist; @@ -305,14 +305,6 @@ struct Fts3Phrase { Fts3Doclist doclist; int bIncr; /* True if doclist is loaded incrementally */ -#if 1 - int isLoaded; /* True if aDoclist/nDoclist are initialized. */ - char *aDoclist; /* Buffer containing doclist */ - int nDoclist; /* Size of aDoclist in bytes */ - sqlite3_int64 iCurrent; - char *pCurrent; -#endif - /* Variables below this point are populated by fts3_expr.c when parsing ** a MATCH expression. Everything above is part of the evaluation phase. */ @@ -380,7 +372,7 @@ void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); -int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); @@ -448,7 +440,6 @@ int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); -int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); /* fts3_tokenizer.c */ @@ -493,13 +484,13 @@ int sqlite3Fts3EvalPhraseDoclist(Fts3Cursor*, Fts3Expr*, const char**,int*); void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr); +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); + int sqlite3Fts3MsrIncrStart( Fts3Table*, Fts3MultiSegReader*, int, const char*, int); int sqlite3Fts3MsrIncrNext( Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); -char *sqlite3Fts3EvalPhrasePoslist( - Fts3Cursor *, Fts3Expr *, sqlite3_int64, int iCol); +char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 44636ed3fc..be40a9cfc2 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -768,10 +768,7 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){ assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); - if( p->pPhrase ){ - sqlite3Fts3EvalPhraseCleanup(p->pPhrase); - sqlite3_free(p->pPhrase->aDoclist); - } + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); sqlite3_free(p); } } diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 187ecbb57f..45e3c32f07 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -176,51 +176,6 @@ static int fts3ExprIterate( return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } -/* -** The argument to this function is always a phrase node. Its doclist -** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes -** to the left of this one in the query tree have already been loaded. -** -** If this phrase node is part of a series of phrase nodes joined by -** NEAR operators (and is not the left-most of said series), then elements are -** removed from the phrases doclist consistent with the NEAR restriction. If -** required, elements may be removed from the doclists of phrases to the -** left of this one that are part of the same series of NEAR operator -** connected phrases. -** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. -*/ -static int fts3ExprNearTrim(Fts3Expr *pExpr){ - int rc = SQLITE_OK; - Fts3Expr *pParent = pExpr->pParent; - - assert( pExpr->eType==FTSQUERY_PHRASE ); - while( rc==SQLITE_OK - && pParent - && pParent->eType==FTSQUERY_NEAR - && pParent->pRight==pExpr - ){ - /* This expression (pExpr) is the right-hand-side of a NEAR operator. - ** Find the expression to the left of the same operator. - */ - int nNear = pParent->nNear; - Fts3Expr *pLeft = pParent->pLeft; - - if( pLeft->eType!=FTSQUERY_PHRASE ){ - assert( pLeft->eType==FTSQUERY_NEAR ); - assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); - pLeft = pLeft->pRight; - } - - rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear); - - pExpr = pLeft; - pParent = pExpr->pParent; - } - - return rc; -} - /* ** This is an fts3ExprIterate() callback used while loading the doclists ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also @@ -236,14 +191,6 @@ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ p->nPhrase++; p->nToken += pPhrase->nToken; - if( pPhrase->isLoaded==0 ){ - rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); - pPhrase->isLoaded = 1; - if( rc==SQLITE_OK ){ - rc = fts3ExprNearTrim(pExpr); - } - } - return rc; } @@ -416,7 +363,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ pPhrase->nToken = pExpr->pPhrase->nToken; - pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->pCsr->iPrevId,p->iCol); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; @@ -867,11 +814,10 @@ static int fts3ExprLocalHitsCb( MatchInfo *p = (MatchInfo *)pCtx; int iStart = iPhrase * p->nCol * 3; int i; - sqlite3_int64 iDocid = p->pCursor->iPrevId; for(i=0; inCol; i++){ char *pCsr; - pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, iDocid, i); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i); if( pCsr ){ p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); }else{ @@ -1020,7 +966,6 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ int i; int iCol; int nToken = 0; - sqlite3_int64 iDocid = pCsr->iPrevId; /* Allocate and populate the array of LcsIterator objects. The array ** contains one element for each matchable phrase in the query. @@ -1042,7 +987,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ for(i=0; inPhrase; i++){ LcsIterator *pIt = &aIter[i]; - pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iDocid, iCol); + pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol); if( pIt->pRead ){ pIt->iPos = pIt->iPosOffset; fts3LcsIteratorAdvance(&aIter[i]); @@ -1396,7 +1341,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iPos = 0; /* First position in position-list */ UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iDocid, p->iCol); + pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index deeff3c418..0470a72960 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -36,6 +36,27 @@ */ #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** TODO: Add a test interface to modify these "constants" from a script for +** this purpose. +*/ +#define FTS3_NODE_CHUNKSIZE (4*1024) +#define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +/* #define FTS3_NODE_CHUNKSIZE 1 */ +/* #define FTS3_NODE_CHUNK_THRESHOLD 1 */ + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; @@ -93,6 +114,9 @@ struct Fts3SegReader { char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + Fts3HashElem **ppNextElem; /* Variables set by fts3SegReaderNext(). These may be read directly @@ -933,7 +957,8 @@ int sqlite3Fts3ReadBlock( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ char **paBlob, /* OUT: Blob data in malloc'd buffer */ - int *pnBlob /* OUT: Size of blob data */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ ){ int rc; /* Return code */ @@ -954,11 +979,16 @@ int sqlite3Fts3ReadBlock( if( rc==SQLITE_OK ){ int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; if( paBlob ){ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); memset(&aByte[nByte], 0, FTS3_NODE_PADDING); if( rc!=SQLITE_OK ){ @@ -968,7 +998,6 @@ int sqlite3Fts3ReadBlock( } *paBlob = aByte; } - *pnBlob = nByte; } return rc; @@ -982,13 +1011,55 @@ void sqlite3Fts3SegmentsClose(Fts3Table *p){ sqlite3_blob_close(p->pSegments); p->pSegments = 0; } + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ -static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ char *pNext; /* Cursor variable */ int nPrefix; /* Number of bytes in term prefix */ int nSuffix; /* Number of bytes in term suffix */ @@ -1000,7 +1071,6 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - int rc; /* Return code from Fts3ReadBlock() */ if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); @@ -1020,6 +1090,8 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; } pReader->aNode = 0; @@ -1031,11 +1103,20 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } rc = sqlite3Fts3ReadBlock( - p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) ); if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } pNext = pReader->aNode; } + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; /* Because of the FTS3_NODE_PADDING bytes of padding, the following is ** safe (no risk of overread) even if the node data is corrupted. @@ -1057,6 +1138,10 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ pReader->zTerm = zNew; pReader->nTermAlloc = nNew; } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; @@ -1069,7 +1154,7 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** of these statements is untrue, then the data structure is corrupt. */ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] - || pReader->aDoclist[pReader->nDoclist-1] + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) ){ return SQLITE_CORRUPT_VTAB; } @@ -1080,12 +1165,16 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** Set the SegReader to point to the first docid in the doclist associated ** with the current term. */ -static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ - int n; +static int fts3SegReaderFirstDocid(Fts3SegReader *pReader){ + int rc; assert( pReader->aDoclist ); assert( !pReader->pOffsetList ); - n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + return rc; } /* @@ -1098,11 +1187,12 @@ static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ ** *pnOffsetList is set to the length of the set of column-offset ** lists, not including the nul-terminator byte. For example: */ -static void fts3SegReaderNextDocid( +static int fts3SegReaderNextDocid( Fts3SegReader *pReader, char **ppOffsetList, int *pnOffsetList ){ + int rc = SQLITE_OK; char *p = pReader->pOffsetList; char c = 0; @@ -1110,7 +1200,16 @@ static void fts3SegReaderNextDocid( ** following two lines advance it to point one byte past the end of ** the same offset list. */ - while( *p | c ) c = *p++ & 0x80; + while( 1 ){ + int nRead; + int rc; + + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + if( pReader->pBlob==0 || (p - pReader->aNode)!=pReader->nPopulate ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } p++; /* If required, populate the output variables with a pointer to and the @@ -1129,10 +1228,15 @@ static void fts3SegReaderNextDocid( if( p>=&pReader->aDoclist[pReader->nDoclist] ){ pReader->pOffsetList = 0; }else{ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - pReader->iDocid += iDelta; + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + pReader->iDocid += iDelta; + } } + + return SQLITE_OK; } /* @@ -1212,7 +1316,7 @@ int sqlite3Fts3SegReaderCost( ** confirms this). */ for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ - rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); + rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob, 0); if( rc!=SQLITE_OK ) break; if( (nBlob+35)>pgsz ){ int nOvfl = (nBlob + 34)/pgsz; @@ -1247,7 +1351,7 @@ int sqlite3Fts3MsrOvfl( int jj; for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ int nBlob; - rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob); + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); if( rc!=SQLITE_OK ) break; if( (nBlob+35)>pgsz ){ nOvfl += (nBlob + 34)/pgsz; @@ -1268,6 +1372,7 @@ void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ sqlite3_free(pReader->zTerm); if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); } } sqlite3_free(pReader); @@ -2193,7 +2298,7 @@ int sqlite3Fts3MsrIncrStart( for(i=0; iapSegment[i]; do { - int rc = fts3SegReaderNext(p, pSeg); + int rc = fts3SegReaderNext(p, pSeg, 1); if( rc!=SQLITE_OK ) return rc; }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } @@ -2210,8 +2315,10 @@ int sqlite3Fts3MsrIncrStart( /* Advance each of the segments to point to the first docid. */ for(i=0; inAdvance; i++){ - fts3SegReaderFirstDocid(pCsr->apSegment[i]); + int rc = fts3SegReaderFirstDocid(pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; } + fts3SegReaderSort(pCsr->apSegment, i, i, fts3SegReaderDoclistCmp); assert( iCol<0 || iColnColumn ); pCsr->iColFilter = iCol; @@ -2226,7 +2333,6 @@ int sqlite3Fts3MsrIncrNext( char **paPoslist, /* OUT: Pointer to position list */ int *pnPoslist /* OUT: Size of position list in bytes */ ){ - int rc = SQLITE_OK; int nMerge = pMsr->nAdvance; Fts3SegReader **apSegment = pMsr->apSegment; @@ -2236,27 +2342,33 @@ int sqlite3Fts3MsrIncrNext( } while( 1 ){ + int nSort; Fts3SegReader *pSeg; - fts3SegReaderSort(pMsr->apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp); pSeg = pMsr->apSegment[0]; if( pSeg->pOffsetList==0 ){ *paPoslist = 0; break; }else{ + int rc; char *pList; int nList; int j; sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + rc = fts3SegReaderNextDocid(apSegment[0], &pList, &nList); j = 1; - while( jpOffsetList && apSegment[j]->iDocid==iDocid ){ fts3SegReaderNextDocid(apSegment[j], 0, 0); + j++; } + if( rc!=SQLITE_OK ) return rc; + + fts3SegReaderSort(pMsr->apSegment, nMerge, j, fts3SegReaderDoclistCmp); if( pMsr->iColFilter>=0 ){ fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); @@ -2269,9 +2381,10 @@ int sqlite3Fts3MsrIncrNext( break; } } + } - return rc; + return SQLITE_OK; } int sqlite3Fts3SegReaderStart( @@ -2295,7 +2408,7 @@ int sqlite3Fts3SegReaderStart( const char *zTerm = pFilter->zTerm; Fts3SegReader *pSeg = pCsr->apSegment[i]; do { - int rc = fts3SegReaderNext(p, pSeg); + int rc = fts3SegReaderNext(p, pSeg, 0); if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } @@ -2331,7 +2444,7 @@ int sqlite3Fts3SegReaderStep( ** forward. Then sort the list in order of current term again. */ for(i=0; inAdvance; i++){ - rc = fts3SegReaderNext(p, apSegment[i]); + rc = fts3SegReaderNext(p, apSegment[i], 0); if( rc!=SQLITE_OK ) return rc; } fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); @@ -2780,15 +2893,6 @@ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ Fts3Phrase *pPhrase = pExpr->pPhrase; fts3DeferredDoclistClear(pExpr->pLeft); fts3DeferredDoclistClear(pExpr->pRight); - if( pPhrase ){ - assert( pExpr->eType==FTSQUERY_PHRASE ); - sqlite3_free(pPhrase->aDoclist); - pPhrase->isLoaded = 0; - pPhrase->aDoclist = 0; - pPhrase->nDoclist = 0; - pPhrase->pCurrent = 0; - pPhrase->iCurrent = 0; - } } } diff --git a/manifest b/manifest index 499805878d..058d53aadb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\simprove\sperformance\sand\ssupport\sLIMIT\sclauses\son\sfts3\stables.\sThis\sbranch\sis\sunstable\sfor\snow. -D 2011-06-02T19:57:24.733 +C FTS\schanges:\sRemove\sunreachable\scode.\sFix\sbugs.\sWhen\sprocessing\sa\slarge\sdoclist\sincrementally,\sread\sfrom\sdisk\sincrementally\stoo. +D 2011-06-03T18:00:19.691 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c f92b6e0241a77a715d30dbeffd7c901053dbfda4 +F ext/fts3/fts3.c 3183cf6aa7bfb00f227be1950b326feb8294da7d F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h ab1489076e7d54714d20bbbc7aaef8e694a7db50 +F ext/fts3/fts3Int.h ba6f831fcde80ed5f0ccb112bb593b117cf11538 F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f -F ext/fts3/fts3_expr.c 5c789c744f98a007512f49d9cda4d2bb4cd56517 +F ext/fts3/fts3_expr.c 0ae554230ada457e61e8184b24faac96aad78f6b F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c 10e0b0995ec82a2d93fbdf3159641cdf30f3c274 +F ext/fts3/fts3_snippet.c 0485969cce410760b50d587a77186f9c7f7e96be F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c b145547430af9451f81cfed92fb7065da2efd870 +F ext/fts3/fts3_write.c c2f041d74f38e40c7487440f78d20f2aaa1f5d3c F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 84097a4c759b1d65890af885f137d3cb16eef584 -R e38e0cc84edbfdb5eae395b3be2f7a86 +P 28149a7882a1e9dfe4a75ec5b91d176ebe6284e9 +R b735e6f33e9ec766163a902e084c30a6 U dan -Z 53d59cf3dcd8991c66b0afd5fb898b1a +Z f3b0ba5ae8b73d0176f1d82c224a912a diff --git a/manifest.uuid b/manifest.uuid index 9e0b1f8e4f..36c53ac073 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28149a7882a1e9dfe4a75ec5b91d176ebe6284e9 \ No newline at end of file +a4c7e2820824e82580730c36f85aede2efa66754 \ No newline at end of file From 5d9c9da6e847e7111fc93c592576edaff5ad8cdf Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Jun 2011 20:11:17 +0000 Subject: [PATCH 27/69] Create and use a function especially for adding the ParseSchema opcode. This gives a small reduction in code and a small performance increase. FossilOrigin-Name: 957b2ab67c6185f0e1062593d237de5c434a38bf --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/alter.c | 4 ++-- src/build.c | 9 ++++----- src/trigger.c | 5 ++--- src/vdbe.h | 1 + src/vdbeaux.c | 21 ++++++++++++++------- src/vtab.c | 2 +- 8 files changed, 36 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index b636720757..c00993097b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\san\s"if"\sout\sof\sa\sloop\sin\sbalance_nonroot()\sfor\sabout\sa\s1%\sperformance\nincrease. -D 2011-06-03T17:50:49.782 +C Create\sand\suse\sa\sfunction\sespecially\sfor\sadding\sthe\sParseSchema\sopcode.\nThis\sgives\sa\ssmall\sreduction\sin\scode\sand\sa\ssmall\sperformance\sincrease. +D 2011-06-03T20:11:17.958 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -115,7 +115,7 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad -F src/alter.c 280f5c04b11b492703a342222b3de0a999445280 +F src/alter.c ac80a0f31189f8b4a524ebf661e47e84536ee7f5 F src/analyze.c a425d62e8fa9ebcb4359ab84ff0c62c6563d2e2a F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 @@ -125,7 +125,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 12aa3b71359c888984223cb2bcf691cf2d7753ae F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c c10ab9e2c77ade99dee23554787f8acfc0c231fc +F src/build.c 5a428625d21ad409514afb40ad083bee25dd957a F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 @@ -231,20 +231,20 @@ F src/test_vfstrace.c 0b884e06094a746da729119a2cabdc7aa790063d F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 7b0c9281b2368dc69135e7a50bd271de9af1b467 -F src/trigger.c 144cc18bb701f3286484aae4292a9531f09278c8 +F src/trigger.c c836a6caac16ba96611558922106858f6ca3d6bf F src/update.c 5bcb56e5c7380a2eecb0e71891dbd4ad7437748f F src/utf.c d83650c3ea08f7407bd9d0839d9885241c209c60 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e F src/vdbe.c edfa3827d7a6fac2425bc10c0eb6e54342d2fa56 -F src/vdbe.h d9c6123384189dc37d27beac1bf44688aa75b6cb +F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1 F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 F src/vdbeapi.c 0eeadc75e44a30efd996d6af6e7c5a2488e35be8 -F src/vdbeaux.c 0505dc4f7ff3cf35e219fe0a20ab798a42772b8b +F src/vdbeaux.c 0b2e2880f13af400a27c92a7673287c3eaec8b21 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 -F src/vtab.c 9ba8c7fdb7d39260c033a402f6032d3e7bc5d336 +F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 0206bc6f87bb9393218a380fc5b18039d334a8d8 -R 9437f6a8af920a11b9b18fbeccb53c6b +P 1bd72d0c616e20fdb395c72ecd96579090ae26cb +R 80ecde493cfcd0f0a571171fe6ad9164 U drh -Z d6270d9f0f6d36a33c5a8dae47c51fc0 +Z 08497a0228f24f1156217359e2dbd164 diff --git a/manifest.uuid b/manifest.uuid index a78d2a2cc4..67c1f2f89d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bd72d0c616e20fdb395c72ecd96579090ae26cb \ No newline at end of file +957b2ab67c6185f0e1062593d237de5c434a38bf \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index aa3fa929f8..fb6d89de6f 100644 --- a/src/alter.c +++ b/src/alter.c @@ -358,14 +358,14 @@ static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ /* Reload the table, index and permanent trigger schemas. */ zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); if( !zWhere ) return; - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); #ifndef SQLITE_OMIT_TRIGGER /* Now, if the table is not stored in the temp database, reload any temp ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. */ if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, 1, zWhere); } #endif } diff --git a/src/build.c b/src/build.c index afe2089325..97758cbcbd 100644 --- a/src/build.c +++ b/src/build.c @@ -1619,8 +1619,8 @@ void sqlite3EndTable( #endif /* Reparse everything to update our internal data structures */ - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "tbl_name='%q'", p->zName)); } @@ -2817,9 +2817,8 @@ Index *sqlite3CreateIndex( if( pTblName ){ sqlite3RefillIndex(pParse, pIndex, iMem); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), - P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); sqlite3VdbeAddOp1(v, OP_Expire, 0); } } diff --git a/src/trigger.c b/src/trigger.c index 0f3f5bad39..6242de5e6c 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -301,9 +301,8 @@ void sqlite3FinishTrigger( pTrig->table, z); sqlite3DbFree(db, z); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, sqlite3MPrintf( - db, "type='trigger' AND name='%q'", zName), P4_DYNAMIC - ); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName)); } if( db->init.busy ){ diff --git a/src/vdbe.h b/src/vdbe.h index 56f9eb51ca..e66ee3024d 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -172,6 +172,7 @@ int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); +void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 8181f01aa3..447ba57a1e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -156,13 +156,6 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ pOp->p3 = p3; pOp->p4.p = 0; pOp->p4type = P4_NOTUSED; - p->expired = 0; - if( op==OP_ParseSchema ){ - /* Any program that uses the OP_ParseSchema opcode needs to lock - ** all btrees. */ - int j; - for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); - } #ifdef SQLITE_DEBUG pOp->zComment = 0; if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]); @@ -201,6 +194,20 @@ int sqlite3VdbeAddOp4( return addr; } +/* +** Add an OP_ParseSchema opcode. This routine is broken out from +** sqlite3VdbeAddOp4() since it needs to also local all btrees. +** +** The zWhere string must have been obtained from sqlite3_malloc(). +** This routine will take ownership of the allocated memory. +*/ +void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ + int j; + int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); + sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); + for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); +} + /* ** Add an opcode that includes the p4 value as an integer. */ diff --git a/src/vtab.c b/src/vtab.c index dffd6a2664..223ef4e7bd 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -383,7 +383,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, pTab->zName, sqlite3Strlen30(pTab->zName) + 1); } From 3a61a5a2b534f357822851d7859a683212aaab72 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Jun 2011 21:34:45 +0000 Subject: [PATCH 28/69] Performance enhancement to the blob-literal tokenizer. FossilOrigin-Name: 61aa2031f1c5ae05e31077588a55194a9546262a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/tokenize.c | 11 +++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index c00993097b..014b8a5783 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Create\sand\suse\sa\sfunction\sespecially\sfor\sadding\sthe\sParseSchema\sopcode.\nThis\sgives\sa\ssmall\sreduction\sin\scode\sand\sa\ssmall\sperformance\sincrease. -D 2011-06-03T20:11:17.958 +C Performance\senhancement\sto\sthe\sblob-literal\stokenizer. +D 2011-06-03T21:34:45.326 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/test_vfs.c e7855568dfa1e0ba73668d273b65605d9f8b77e8 F src/test_vfstrace.c 0b884e06094a746da729119a2cabdc7aa790063d F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/tokenize.c 7b0c9281b2368dc69135e7a50bd271de9af1b467 +F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 F src/trigger.c c836a6caac16ba96611558922106858f6ca3d6bf F src/update.c 5bcb56e5c7380a2eecb0e71891dbd4ad7437748f F src/utf.c d83650c3ea08f7407bd9d0839d9885241c209c60 @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 1bd72d0c616e20fdb395c72ecd96579090ae26cb -R 80ecde493cfcd0f0a571171fe6ad9164 +P 957b2ab67c6185f0e1062593d237de5c434a38bf +R bd36ddd3c30f1348cf18f2599b20bbca U drh -Z 08497a0228f24f1156217359e2dbd164 +Z fe6a860af7a1658c61a1531782a69559 diff --git a/manifest.uuid b/manifest.uuid index 67c1f2f89d..01e044ed3e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -957b2ab67c6185f0e1062593d237de5c434a38bf \ No newline at end of file +61aa2031f1c5ae05e31077588a55194a9546262a \ No newline at end of file diff --git a/src/tokenize.c b/src/tokenize.c index 016a218262..b32989277e 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -353,13 +353,12 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ testcase( z[0]=='x' ); testcase( z[0]=='X' ); if( z[1]=='\'' ){ *tokenType = TK_BLOB; - for(i=2; (c=z[i])!=0 && c!='\''; i++){ - if( !sqlite3Isxdigit(c) ){ - *tokenType = TK_ILLEGAL; - } + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } } - if( i%2 || !c ) *tokenType = TK_ILLEGAL; - if( c ) i++; + if( z[i] ) i++; return i; } /* Otherwise fall through to the next case */ From 61d2fe955c2af8a24ab3829d143c8f9fd6324319 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Jun 2011 23:28:33 +0000 Subject: [PATCH 29/69] Performance improvements on memory copies inside of btree by moving 2 bytes at a time instead of just 1 byte at a time. FossilOrigin-Name: 897f56a158ebe62758c9998e4941ae046c75fb99 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 17 ++++++++++------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 014b8a5783..7f9a970797 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\senhancement\sto\sthe\sblob-literal\stokenizer. -D 2011-06-03T21:34:45.326 +C Performance\simprovements\son\smemory\scopies\sinside\sof\sbtree\sby\smoving\s2\sbytes\nat\sa\stime\sinstead\sof\sjust\s1\sbyte\sat\sa\stime. +D 2011-06-03T23:28:33.011 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,7 +122,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 12aa3b71359c888984223cb2bcf691cf2d7753ae +F src/btree.c 9f214af284fae1e96f12989dcc4d26f247b502fc F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 5a428625d21ad409514afb40ad083bee25dd957a @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 957b2ab67c6185f0e1062593d237de5c434a38bf -R bd36ddd3c30f1348cf18f2599b20bbca +P 61aa2031f1c5ae05e31077588a55194a9546262a +R ee8453a74951d0e43529f6f62862f77e U drh -Z fe6a860af7a1658c61a1531782a69559 +Z eb5472e3af1761c7727508ef6d936f22 diff --git a/manifest.uuid b/manifest.uuid index 01e044ed3e..246d28ad64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61aa2031f1c5ae05e31077588a55194a9546262a \ No newline at end of file +897f56a158ebe62758c9998e4941ae046c75fb99 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9e9677cc42..ece4602159 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5422,8 +5422,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ } endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2]; while( ptrnCell--; @@ -5464,6 +5463,7 @@ static void insertCell( int cellOffset; /* Address of first cell pointer in data[] */ u8 *data; /* The content of the whole page */ u8 *ptr; /* Used for moving information around in data[] */ + u8 *endPtr; /* End of the loop */ int nSkip = (iChild ? 4 : 0); @@ -5514,9 +5514,11 @@ static void insertCell( if( iChild ){ put4byte(&data[idx], iChild); } - for(j=end, ptr=&data[j]; j>ins; j-=2, ptr-=2){ - ptr[0] = ptr[-2]; - ptr[1] = ptr[-1]; + ptr = &data[end]; + endPtr = &data[ins]; + while( ptr>endPtr ){ + *(u16*)ptr = *(u16*)&ptr[-2]; + ptr -= 2; } put2byte(&data[ins], idx); put2byte(&data[pPage->hdrOffset+3], pPage->nCell); @@ -5561,10 +5563,11 @@ static void assemblePage( pCellptr = &data[pPage->cellOffset + nCell*2]; cellbody = nUsable; for(i=nCell-1; i>=0; i--){ + u16 sz = aSize[i]; pCellptr -= 2; - cellbody -= aSize[i]; + cellbody -= sz; put2byte(pCellptr, cellbody); - memcpy(&data[cellbody], apCell[i], aSize[i]); + memcpy(&data[cellbody], apCell[i], sz); } put2byte(&data[hdr+3], nCell); put2byte(&data[hdr+5], cellbody); From afb9817f8c85b9a3e43edabfc84e079c8dd384ab Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 Jun 2011 01:43:53 +0000 Subject: [PATCH 30/69] Performance improvement to the btree search routine. FossilOrigin-Name: 65db822f200bafe9abe59b33b17b2c643c17c5e8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 7f9a970797..acedbbc719 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovements\son\smemory\scopies\sinside\sof\sbtree\sby\smoving\s2\sbytes\nat\sa\stime\sinstead\sof\sjust\s1\sbyte\sat\sa\stime. -D 2011-06-03T23:28:33.011 +C Performance\simprovement\sto\sthe\sbtree\ssearch\sroutine. +D 2011-06-04T01:43:53.858 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,7 +122,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 9f214af284fae1e96f12989dcc4d26f247b502fc +F src/btree.c c19d7b35c8325b302d88d73e834642a7a96ee003 F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 5a428625d21ad409514afb40ad083bee25dd957a @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 61aa2031f1c5ae05e31077588a55194a9546262a -R ee8453a74951d0e43529f6f62862f77e +P 897f56a158ebe62758c9998e4941ae046c75fb99 +R 84139bde7a129a9544042a4a85d415c4 U drh -Z eb5472e3af1761c7727508ef6d936f22 +Z 4316427fea8d5aeb84c01ffb6d821ecd diff --git a/manifest.uuid b/manifest.uuid index 246d28ad64..e0e3846b64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -897f56a158ebe62758c9998e4941ae046c75fb99 \ No newline at end of file +65db822f200bafe9abe59b33b17b2c643c17c5e8 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index ece4602159..640d48b452 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4453,7 +4453,7 @@ int sqlite3BtreeMovetoUnpacked( } assert( pCur->apPage[0]->intKey || pIdxKey ); for(;;){ - int lwr, upr; + int lwr, upr, idx; Pgno chldPg; MemPage *pPage = pCur->apPage[pCur->iPage]; int c; @@ -4469,14 +4469,14 @@ int sqlite3BtreeMovetoUnpacked( lwr = 0; upr = pPage->nCell-1; if( biasRight ){ - pCur->aiIdx[pCur->iPage] = (u16)upr; + pCur->aiIdx[pCur->iPage] = (u16)(idx = upr); }else{ - pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2); + pCur->aiIdx[pCur->iPage] = (u16)(idx = (upr+lwr)/2); } for(;;){ - int idx = pCur->aiIdx[pCur->iPage]; /* Index of current cell in pPage */ u8 *pCell; /* Pointer to current cell in pPage */ + assert( idx==pCur->aiIdx[pCur->iPage] ); pCur->info.nSize = 0; pCell = findCell(pPage, idx) + pPage->childPtrSize; if( pPage->intKey ){ @@ -4559,7 +4559,7 @@ int sqlite3BtreeMovetoUnpacked( if( lwr>upr ){ break; } - pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2); + pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2); } assert( lwr==upr+1 ); assert( pPage->isInit ); From b46ee9172942d0ef12cbb14461b5fb1fc1367c8b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Jun 2011 20:04:35 +0000 Subject: [PATCH 31/69] Allow the "order=DESC" and "order=ASC" parameters in FTS4 "CREATE VIRTUAL TABLE" statements. Tables created with "order=DESC" store all doclists in descending order, which allows optimizations normally applied to "ORDER BY docid ASC" queries to be used with "ORDER BY docid DESC" queries instead. FossilOrigin-Name: f6a0193f5a32603eb48bddc6297042dbd2ffe96e --- ext/fts3/fts3.c | 558 ++++++++++++++++++++++++++++++------------ ext/fts3/fts3Int.h | 5 +- ext/fts3/fts3_write.c | 245 ++++++++++++------- manifest | 18 +- manifest.uuid | 2 +- test/fts3sort.test | 157 ++++++++---- 6 files changed, 682 insertions(+), 303 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 90b081b8c5..695d3e2d5e 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -423,12 +423,12 @@ static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ ** When this function is called, *pp points to the first byte following a ** varint that is part of a doclist (or position-list, or any other list ** of varints). This function moves *pp to point to the start of that varint, -** and decrements the value stored in *pVal by the varint value. +** and sets *pVal by the varint value. ** ** Argument pStart points to the first byte of the doclist that the ** varint is part of. */ -static void fts3GetReverseDeltaVarint( +static void fts3GetReverseVarint( char **pp, char *pStart, sqlite3_int64 *pVal @@ -444,7 +444,7 @@ static void fts3GetReverseDeltaVarint( *pp = p; sqlite3Fts3GetVarint(p, &iVal); - *pVal -= iVal; + *pVal = iVal; } /* @@ -916,17 +916,37 @@ static int fts3InitVtab( int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ - int bNoDocsize = 0; /* True to omit %_docsize table */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ - char *zPrefix = 0; /* Prefix parameter value (or NULL) */ int nIndex; /* Size of aIndex[] array */ struct Fts3Index *aIndex; /* Array of indexes for this table */ struct Fts3Index *aFree = 0; /* Free this before returning */ - char *zCompress = 0; - char *zUncompress = 0; + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + + char *zMatchinfo = 0; /* Prefix parameter value (or NULL) */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zOrder = 0; /* order=? parameter (or NULL) */ + struct Fts4Option { + const char *zOpt; + int nOpt; + char **pzVar; + } aFts4Opt[] = { + { "matchinfo", 9, 0 }, + { "prefix", 6, 0 }, + { "compress", 8, 0 }, + { "uncompress", 10, 0 }, + { "order", 5, 0 } + }; + aFts4Opt[0].pzVar = &zMatchinfo; + aFts4Opt[1].pzVar = &zPrefix; + aFts4Opt[2].pzVar = &zCompress; + aFts4Opt[3].pzVar = &zUncompress; + aFts4Opt[4].pzVar = &zOrder; assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) @@ -967,32 +987,27 @@ static int fts3InitVtab( /* Check if it is an FTS4 special argument. */ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + int iOpt; if( !zVal ){ rc = SQLITE_NOMEM; - goto fts3_init_out; - } - if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ - if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ - bNoDocsize = 1; - }else{ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ - zCompress = zVal; - zVal = 0; - }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ - zUncompress = zVal; - zVal = 0; - }else if( nKey==6 && 0==sqlite3_strnicmp(z, "prefix", 6) ){ - sqlite3_free(zPrefix); - zPrefix = zVal; - zVal = 0; }else{ - *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); - rc = SQLITE_ERROR; + for(iOpt=0; iOptnMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; + p->bDescIdx = bDescIdx; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); @@ -1108,6 +1144,8 @@ fts3_init_out: sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); + sqlite3_free(zOrder); + sqlite3_free(zMatchinfo); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ @@ -1880,10 +1918,11 @@ static int fts3PoslistNearMerge( #define MERGE_NOT 2 /* D + D -> D */ #define MERGE_AND 3 /* D + D -> D */ #define MERGE_OR 4 /* D + D -> D */ -#define MERGE_POS_OR 5 /* P + P -> P */ #define MERGE_PHRASE 6 /* P + P -> D */ -#define MERGE_POS_PHRASE 7 /* P + P -> P */ #define MERGE_NEAR 8 /* P + P -> D */ + +#define MERGE_POS_OR 5 /* P + P -> P */ +#define MERGE_POS_PHRASE 7 /* P + P -> P */ #define MERGE_POS_NEAR 9 /* P + P -> P */ /* @@ -1924,6 +1963,7 @@ static int fts3DoclistMerge( || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR ); + assert( mergetype==MERGE_POS_PHRASE || mergetype==MERGE_POS_NEAR ); if( !aBuffer ){ *pnBuffer = 0; @@ -2065,6 +2105,227 @@ struct TermSelect { int anOutput[16]; /* Size of output in bytes */ }; + +static void fts3GetDeltaVarint3( + char **pp, + char *pEnd, + int bDescIdx, + sqlite3_int64 *pVal +){ + if( *pp>=pEnd ){ + *pp = 0; + }else{ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; + }else{ + *pVal += iVal; + } + } +} + +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + +#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) + +static int fts3DoclistOrMerge( + int bDescIdx, /* True if arguments are desc */ + u8 *a1, int n1, /* First doclist */ + u8 *a2, int n2, /* Second doclist */ + u8 **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + int nOut; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + aOut = sqlite3_malloc(n1+n2); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *paOut = aOut; + *pnOut = (p-aOut); + return SQLITE_OK; +} + +static void fts3DoclistPhraseMerge( + int bDescIdx, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + u8 *aLeft, int nLeft, /* Left doclist */ + u8 *aRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut = aRight; + + assert( nDist>0 ); + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *pnRight = p - aOut; +} + +/* +** This function merges two doclists according to the requirements of a +** NEAR operator. +*/ +static int fts3DoclistNearMerge( + int bDescIdx, + int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ + int nNear, /* Parameter to NEAR operator */ + int nTokenLeft, /* Number of tokens in LHS phrase arg */ + char *aLeft, /* Doclist for LHS (incl. positions) */ + int nLeft, /* Size of LHS doclist in bytes */ + int nTokenRight, /* As nTokenLeft */ + char *aRight, /* As aLeft */ + int nRight, /* As nRight */ + char **paOut, /* OUT: Results of merge (malloced) */ + int *pnOut /* OUT: Sized of output buffer */ +){ + char *aOut; /* Buffer to write output doclist to */ + char *aTmp; /* Temp buffer used by PoslistNearMerge() */ + + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + int bFirstOut = 0; + + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[nRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + + int nParam1 = nNear+nTokenRight; + int nParam2 = nNear+nTokenLeft; + + p = aOut = sqlite3_malloc(nLeft+nRight+1); + aTmp = sqlite3_malloc(2*(nLeft+nRight+1)); + if( !aOut || !aTmp ){ + sqlite3_free(aOut); + sqlite3_free(aTmp); + *paOut = 0; + *pnOut = 0; + return SQLITE_NOMEM; + } + + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + if( !fts3PoslistNearMerge(&p, aTmp, nParam1, nParam2, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + sqlite3_free(aTmp); + *paOut = aOut; + *pnOut = p - aOut; + return SQLITE_OK; +} + + + /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all @@ -2074,7 +2335,7 @@ struct TermSelect { ** the responsibility of the caller to free any doclists left in the ** TermSelect.aaOutput[] array. */ -static int fts3TermSelectMerge(TermSelect *pTS){ +static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); char *aOut = 0; int nOut = 0; @@ -2090,15 +2351,17 @@ static int fts3TermSelectMerge(TermSelect *pTS){ nOut = pTS->anOutput[i]; pTS->aaOutput[i] = 0; }else{ - int nNew = nOut + pTS->anOutput[i]; - char *aNew = sqlite3_malloc(nNew); - if( !aNew ){ - sqlite3_free(aOut); - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 + int nNew; + u8 *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + sqlite3_free(pTS->aaOutput[i]); sqlite3_free(aOut); pTS->aaOutput[i] = 0; @@ -2134,9 +2397,7 @@ static int fts3TermSelectCb( if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). TODO: Add a way to transfer control of the - ** aDoclist buffer from the caller so as to avoid the memcpy(). - */ + ** buffer using memcpy(). */ pTS->aaOutput[0] = sqlite3_malloc(nDoclist); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ @@ -2151,36 +2412,33 @@ static int fts3TermSelectCb( int iOut; for(iOut=0; iOutaaOutput); iOut++){ - char *aNew; - int nNew; if( pTS->aaOutput[iOut]==0 ){ assert( iOut>0 ); pTS->aaOutput[iOut] = aMerge; pTS->anOutput[iOut] = nMerge; break; - } + }else{ + u8 *aNew; + int nNew; - nNew = nMerge + pTS->anOutput[iOut]; - aNew = sqlite3_malloc(nNew); - if( !aNew ){ - if( aMerge!=aDoclist ){ - sqlite3_free(aMerge); + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; } - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, - pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 - ); - if( iOut>0 ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } } } } @@ -2426,7 +2684,7 @@ static int fts3TermSelect( } if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(&tsc); + rc = fts3TermSelectMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; @@ -2476,48 +2734,6 @@ static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ return nDoc; } -/* -** This function merges two doclists according to the requirements of a -** NEAR operator. -** -** Both input doclists must include position information. The output doclist -** includes position information if the first argument to this function -** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. -*/ -static int fts3NearMerge( - int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ - int nNear, /* Parameter to NEAR operator */ - int nTokenLeft, /* Number of tokens in LHS phrase arg */ - char *aLeft, /* Doclist for LHS (incl. positions) */ - int nLeft, /* Size of LHS doclist in bytes */ - int nTokenRight, /* As nTokenLeft */ - char *aRight, /* As aLeft */ - int nRight, /* As nRight */ - char **paOut, /* OUT: Results of merge (malloced) */ - int *pnOut /* OUT: Sized of output buffer */ -){ - char *aOut; /* Buffer to write output doclist to */ - int rc; /* Return code */ - - assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); - - aOut = sqlite3_malloc(nLeft+nRight+1); - if( aOut==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, - aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - aOut = 0; - } - } - - *paOut = aOut; - return rc; -} - /* ** Advance the cursor to the next row in the %_content table that ** matches the search criteria. For a MATCH search, this will be @@ -2588,7 +2804,11 @@ static int fts3FilterMethod( sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); - pCsr->bDesc = (idxStr && idxStr[0]=='D'); + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } pCsr->eSearch = (i16)idxNum; if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ @@ -2627,7 +2847,7 @@ static int fts3FilterMethod( ** row by docid. */ if( idxNum==FTS3_FULLSCAN_SEARCH ){ - const char *zSort = (idxStr ? idxStr : "ASC"); + const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); }else{ @@ -3265,8 +3485,8 @@ static int fts3EvalPhraseLoad( nDoclist = nThis; }else{ assert( iPrev>=0 ); - fts3DoclistMerge(MERGE_POS_PHRASE, iToken-iPrev, - 0, pThis, &nThis, aDoclist, nDoclist, pThis, nThis, 0 + fts3DoclistPhraseMerge(pTab->bDescIdx, + iToken-iPrev, aDoclist, nDoclist, pThis, &nThis ); sqlite3_free(aDoclist); aDoclist = pThis; @@ -3411,14 +3631,14 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ int rc; Fts3Doclist *pList = &p->doclist; Fts3PhraseToken *pFirst = &p->aToken[0]; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; assert( pList->aAll==0 ); - if( pCsr->bDesc==0 && bOptOk==1 && p->nToken==1 + if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup ){ /* Use the incremental approach. */ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); rc = sqlite3Fts3MsrIncrStart( pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); @@ -3434,6 +3654,58 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ return rc; } +void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* IN/OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + int iMul = (bDescIdx ? -1 : 1); + + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + + pDocid += sqlite3Fts3GetVarint(pDocid, &iDocid); + pNext = pDocid; + fts3PoslistCopy(0, &pDocid); + while( pDociddoclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; if( p->bIncr ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; assert( p->nToken==1 ); rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, &pDL->iDocid, &pDL->pList, &pDL->nList @@ -3460,42 +3732,12 @@ static int fts3EvalPhraseNext( if( rc==SQLITE_OK && !pDL->pList ){ *pbEof = 1; } - }else if( pCsr->bDesc && pDL->aAll ){ - - if( pDL->pNextDocid==0 ){ - sqlite3_int64 iDocid = 0; - char *pNext; - char *pDocid = pDL->aAll; - char *pEnd = &pDocid[pDL->nAll]; - - while( pDocidpNextDocid = pDocid; - pDL->pList = pDocid; - fts3PoslistCopy(0, &pDocid); - } - pDL->nList = (pEnd - pDL->pList); - pDL->iDocid = iDocid; - }else{ - - assert( *pbEof==0 ); - assert( pDL->pNextDocid>pDL->aAll ); - - fts3GetReverseDeltaVarint( - &pDL->pNextDocid, pDL->aAll, &pDL->iDocid - ); - if( pDL->pNextDocid==pDL->aAll ){ - *pbEof = 1; - }else{ - char *pSave = pDL->pNextDocid; - fts3ReversePoslist(pDL->aAll, &pDL->pNextDocid); - pDL->pList = pDL->pNextDocid; - pDL->nList = pSave - pDL->pNextDocid; - } - } - + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->aAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; }else{ - char *pIter; if( pDL->pNextDocid ){ pIter = pDL->pNextDocid; @@ -3507,7 +3749,13 @@ static int fts3EvalPhraseNext( /* We have already reached the end of this doclist. EOF. */ *pbEof = 1; }else{ - fts3GetDeltaVarint(&pIter, &pDL->iDocid); + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } pDL->pList = pIter; fts3PoslistCopy(0, &pIter); pDL->nList = (pIter - pDL->pList); @@ -3546,6 +3794,7 @@ static void fts3EvalStartReaders( } static void fts3EvalNearMerge( + int bDescIdx, Fts3Expr *p1, Fts3Expr *p2, int nNear, @@ -3567,7 +3816,7 @@ static void fts3EvalNearMerge( char *aOut; /* Buffer in which to assemble new doclist */ int nOut; /* Size of buffer aOut in bytes */ - *pRc = fts3NearMerge(MERGE_POS_NEAR, nNear, + *pRc = fts3DoclistNearMerge(bDescIdx, MERGE_POS_NEAR, nNear, pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll, pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll, &aOut, &nOut @@ -3602,6 +3851,7 @@ static void fts3EvalNearTrim(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ if( !aPhrase ){ *pRc = SQLITE_NOMEM; }else{ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int i = 1; aPhrase[0] = pLeft; do { @@ -3611,11 +3861,11 @@ static void fts3EvalNearTrim(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ for(i=0; i<(nPhrase-1); i++){ int nNear = aPhrase[i+1]->pParent->nNear; - fts3EvalNearMerge(aPhrase[i], aPhrase[i+1], nNear, pRc); + fts3EvalNearMerge(p->bDescIdx, aPhrase[i], aPhrase[i+1], nNear, pRc); } for(i=nPhrase-2; i>=0; i--){ int nNear = aPhrase[i+1]->pParent->nNear; - fts3EvalNearMerge(aPhrase[i+1], aPhrase[i], nNear, pRc); + fts3EvalNearMerge(p->bDescIdx, aPhrase[i+1], aPhrase[i], nNear, pRc); } sqlite3_free(aPhrase); diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index c6834d37b6..c6329c30d0 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -180,6 +180,7 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ @@ -236,7 +237,7 @@ struct Fts3Cursor { char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ - int bDesc; /* True to sort in descending order */ + u8 bDesc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ @@ -438,6 +439,7 @@ int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); +void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); @@ -495,4 +497,5 @@ int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); + #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 0470a72960..fd964739b2 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -62,9 +62,8 @@ typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; /* -** Data structure used while accumulating terms in the pending-terms hash -** table. The hash table entry maps from term (a string) to a malloc'd -** instance of this structure. +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { int nData; @@ -130,8 +129,11 @@ struct Fts3SegReader { char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ - /* The following variables are used to iterate through the current doclist */ + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ sqlite3_int64 iDocid; }; @@ -573,11 +575,21 @@ static int fts3PendingListAppend( return 0; } +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ static int fts3PendingTermsAddOne( Fts3Table *p, int iCol, int iPos, - Fts3Hash *pHash, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ const char *zToken, int nToken ){ @@ -713,7 +725,8 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ Fts3HashElem *pElem; Fts3Hash *pHash = &p->aIndex[i].hPending; for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); } fts3HashClear(pHash); } @@ -1115,12 +1128,13 @@ static int fts3SegReaderNext( pNext = pReader->aNode; } + assert( !fts3SegReaderIsPending(pReader) ); + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); if( rc!=SQLITE_OK ) return rc; /* Because of the FTS3_NODE_PADDING bytes of padding, the following is - ** safe (no risk of overread) even if the node data is corrupted. - */ + ** safe (no risk of overread) even if the node data is corrupted. */ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); if( nPrefix<0 || nSuffix<=0 @@ -1165,14 +1179,24 @@ static int fts3SegReaderNext( ** Set the SegReader to point to the first docid in the doclist associated ** with the current term. */ -static int fts3SegReaderFirstDocid(Fts3SegReader *pReader){ - int rc; +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; assert( pReader->aDoclist ); assert( !pReader->pOffsetList ); - rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); - if( rc==SQLITE_OK ){ - int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } } return rc; } @@ -1188,51 +1212,83 @@ static int fts3SegReaderFirstDocid(Fts3SegReader *pReader){ ** lists, not including the nul-terminator byte. For example: */ static int fts3SegReaderNextDocid( - Fts3SegReader *pReader, - char **ppOffsetList, - int *pnOffsetList + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ ){ int rc = SQLITE_OK; char *p = pReader->pOffsetList; char c = 0; - /* Pointer p currently points at the first byte of an offset list. The - ** following two lines advance it to point one byte past the end of - ** the same offset list. - */ - while( 1 ){ - int nRead; - int rc; + assert( p ); - while( *p | c ) c = *p++ & 0x80; - assert( *p==0 ); - if( pReader->pBlob==0 || (p - pReader->aNode)!=pReader->nPopulate ) break; - rc = fts3SegReaderIncrRead(pReader); - if( rc!=SQLITE_OK ) return rc; - } - p++; - - /* If required, populate the output variables with a pointer to and the - ** size of the previous offset-list. - */ - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = (int)(p - pReader->pOffsetList - 1); - } - - /* If there are no more entries in the doclist, set pOffsetList to - ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and - ** Fts3SegReader.pOffsetList to point to the next offset list before - ** returning. - */ - if( p>=&pReader->aDoclist[pReader->nDoclist] ){ - pReader->pOffsetList = 0; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } }else{ - rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); - if( rc==SQLITE_OK ){ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - pReader->iDocid += iDelta; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte passed + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + /* If there are no more entries in the doclist, set pOffsetList to + ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and + ** Fts3SegReader.pOffsetList to point to the next offset list before + ** returning. + */ + if( p>=&pReader->aDoclist[pReader->nDoclist] ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid -= iDelta; + }else{ + pReader->iDocid += iDelta; + } + } } } @@ -1601,6 +1657,18 @@ static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ assert( pLhs->aNode && pRhs->aNode ); return rc; } +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} /* ** Compare the term that the Fts3SegReader object passed as the first argument @@ -2290,6 +2358,9 @@ int sqlite3Fts3MsrIncrStart( ){ int i; int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); assert( pCsr->pFilter==0 ); assert( zTerm && nTerm>0 ); @@ -2315,10 +2386,10 @@ int sqlite3Fts3MsrIncrStart( /* Advance each of the segments to point to the first docid. */ for(i=0; inAdvance; i++){ - int rc = fts3SegReaderFirstDocid(pCsr->apSegment[i]); + int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); if( rc!=SQLITE_OK ) return rc; } - fts3SegReaderSort(pCsr->apSegment, i, i, fts3SegReaderDoclistCmp); + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); assert( iCol<0 || iColnColumn ); pCsr->iColFilter = iCol; @@ -2335,6 +2406,9 @@ int sqlite3Fts3MsrIncrNext( ){ int nMerge = pMsr->nAdvance; Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); if( nMerge==0 ){ *paPoslist = 0; @@ -2356,19 +2430,18 @@ int sqlite3Fts3MsrIncrNext( int j; sqlite3_int64 iDocid = apSegment[0]->iDocid; - rc = fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; while( rc==SQLITE_OK && jpOffsetList && apSegment[j]->iDocid==iDocid ){ - fts3SegReaderNextDocid(apSegment[j], 0, 0); + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); j++; } if( rc!=SQLITE_OK ) return rc; - - fts3SegReaderSort(pMsr->apSegment, nMerge, j, fts3SegReaderDoclistCmp); + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); if( pMsr->iColFilter>=0 ){ fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); @@ -2433,6 +2506,9 @@ int sqlite3Fts3SegReaderStep( Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); if( pCsr->nSegment==0 ) return SQLITE_OK; @@ -2483,7 +2559,10 @@ int sqlite3Fts3SegReaderStep( } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); - if( nMerge==1 && !isIgnoreEmpty ){ + if( nMerge==1 + && !isIgnoreEmpty + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ pCsr->aDoclist = apSegment[0]->aDoclist; pCsr->nDoclist = apSegment[0]->nDoclist; rc = SQLITE_ROW; @@ -2496,22 +2575,22 @@ int sqlite3Fts3SegReaderStep( ** and a single term returned with the merged doclist. */ for(i=0; ipOffsetList ){ int j; /* Number of segments that share a docid */ char *pList; int nList; int nByte; sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; while( jpOffsetList && apSegment[j]->iDocid==iDocid ){ - fts3SegReaderNextDocid(apSegment[j], 0, 0); + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); j++; } @@ -2520,7 +2599,19 @@ int sqlite3Fts3SegReaderStep( } if( !isIgnoreEmpty || nList>0 ){ - nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + iDelta = iPrev - iDocid; + }else{ + iDelta = iDocid - iPrev; + } + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); + assert( nDoclist>0 || iDelta==iDocid ); + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); if( nDoclist+nByte>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = (nDoclist+nByte)*2; @@ -2530,9 +2621,7 @@ int sqlite3Fts3SegReaderStep( } pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint( - &pCsr->aBuffer[nDoclist], iDocid-iPrev - ); + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); iPrev = iDocid; if( isRequirePos ){ memcpy(&pCsr->aBuffer[nDoclist], pList, nList); @@ -2541,7 +2630,7 @@ int sqlite3Fts3SegReaderStep( } } - fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); + fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ pCsr->aDoclist = pCsr->aBuffer; @@ -2883,19 +2972,6 @@ char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ return 0; } -/* -** Helper fucntion for FreeDeferredDoclists(). This function removes all -** references to deferred doclists from within the tree of Fts3Expr -** structures headed by -*/ -static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3DeferredDoclistClear(pExpr->pLeft); - fts3DeferredDoclistClear(pExpr->pRight); - } -} - /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. @@ -2903,12 +2979,9 @@ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ Fts3DeferredToken *pDef; for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); pDef->pList = 0; } - if( pCsr->pDeferred ){ - fts3DeferredDoclistClear(pCsr->pExpr); - } } /* @@ -2920,7 +2993,7 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ Fts3DeferredToken *pNext; for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ pNext = pDef->pNext; - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); sqlite3_free(pDef); } pCsr->pDeferred = 0; diff --git a/manifest b/manifest index 058d53aadb..7bf78b4a1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C FTS\schanges:\sRemove\sunreachable\scode.\sFix\sbugs.\sWhen\sprocessing\sa\slarge\sdoclist\sincrementally,\sread\sfrom\sdisk\sincrementally\stoo. -D 2011-06-03T18:00:19.691 +C Allow\sthe\s"order=DESC"\sand\s"order=ASC"\sparameters\sin\sFTS4\s"CREATE\sVIRTUAL\sTABLE"\sstatements.\sTables\screated\swith\s"order=DESC"\sstore\sall\sdoclists\sin\sdescending\sorder,\swhich\sallows\soptimizations\snormally\sapplied\sto\s"ORDER\sBY\sdocid\sASC"\squeries\sto\sbe\sused\swith\s"ORDER\sBY\sdocid\sDESC"\squeries\sinstead. +D 2011-06-04T20:04:35.492 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,9 +61,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 3183cf6aa7bfb00f227be1950b326feb8294da7d +F ext/fts3/fts3.c 432c902c697850cbc5ffc2be8a945ac7ac3328d7 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h ba6f831fcde80ed5f0ccb112bb593b117cf11538 +F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f F ext/fts3/fts3_expr.c 0ae554230ada457e61e8184b24faac96aad78f6b F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c @@ -75,7 +75,7 @@ F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c c2f041d74f38e40c7487440f78d20f2aaa1f5d3c +F ext/fts3/fts3_write.c d7eded6f5ee3032b41126047cc04b6720f59e6da F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -478,7 +478,7 @@ F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd -F test/fts3sort.test e6f24e9cffc46484bcc9fe63d3c2ce41afcaa6c9 +F test/fts3sort.test c599f19e9452b5735c41889ea5a6da996f0ebc7c F test/fts4aa.test eadf85621c0a113d4c7ad3ccbf8441130e007b8f F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 28149a7882a1e9dfe4a75ec5b91d176ebe6284e9 -R b735e6f33e9ec766163a902e084c30a6 +P a4c7e2820824e82580730c36f85aede2efa66754 +R 2bf28954217e137f11160d52b346ca5f U dan -Z f3b0ba5ae8b73d0176f1d82c224a912a +Z b16ce48fbf2f9ba7236a3b596adb6f59 diff --git a/manifest.uuid b/manifest.uuid index 36c53ac073..2d5316d141 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4c7e2820824e82580730c36f85aede2efa66754 \ No newline at end of file +f6a0193f5a32603eb48bddc6297042dbd2ffe96e \ No newline at end of file diff --git a/test/fts3sort.test b/test/fts3sort.test index 001bef1135..d2edbc2ade 100644 --- a/test/fts3sort.test +++ b/test/fts3sort.test @@ -21,9 +21,8 @@ ifcapable !fts3 { return } -set testprefix fts3sort -proc build_database {nRow} { +proc build_database {nRow param} { db close forcedelete test.db sqlite3 db test.db @@ -31,7 +30,7 @@ proc build_database {nRow} { set vocab [list aa ab ac ba bb bc ca cb cc da] expr srand(0) - execsql { CREATE VIRTUAL TABLE t1 USING fts4 } + execsql "CREATE VIRTUAL TABLE t1 USING fts4($param)" for {set i 0} {$i < $nRow} {incr i} { set v [expr int(rand()*1000000)] set doc [list] @@ -42,13 +41,24 @@ proc build_database {nRow} { } } -set nRow 1000 -do_test 1.0 { - build_database $nRow - execsql { SELECT count(*) FROM t1 } -} $nRow +set testprefix fts3sort -foreach {tn query} { +unset -nocomplain CONTROL +foreach {t param} { + 1 "" + 2 "order=asc" + 3 "order=desc" +} { + + set testprefix fts3sort-1.$t + + set nRow 1000 + do_test 1.0 { + build_database $nRow $param + execsql { SELECT count(*) FROM t1 } + } $nRow + + foreach {tn query} { 1 "SELECT docid, * FROM t1" 2 "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'" 3 "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'" @@ -59,50 +69,93 @@ foreach {tn query} { 8 "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'" 9 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'" 10 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR nosuchtoken'" + 11 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa NEAR bb'" + 12 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH '\"aa bb\"'" + 13 "SELECT docid, content FROM t1 WHERE t1 MATCH 'aa NEAR/2 bb NEAR/3 cc'" + 14 "SELECT docid, content FROM t1 WHERE t1 MATCH '\"aa bb cc\"'" + } { + + unset -nocomplain A B C D + set A_list [list] + set B_list [list] + set C_list [list] + set D_list [list] + + unset -nocomplain X + db eval "$query ORDER BY rowid ASC" X { + set A($X(docid)) [array get X] + lappend A_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY rowid DESC" X { + set B($X(docid)) [array get X] + lappend B_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid ASC" X { + set C($X(docid)) [array get X] + lappend C_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid DESC" X { + set D($X(docid)) [array get X] + lappend D_list $X(docid) + } + + do_test $tn.1 { set A_list } [lsort -integer -increasing $A_list] + do_test $tn.2 { set B_list } [lsort -integer -decreasing $B_list] + do_test $tn.3 { set C_list } [lsort -integer -increasing $C_list] + do_test $tn.4 { set D_list } [lsort -integer -decreasing $D_list] + + unset -nocomplain DATA + unset -nocomplain X + db eval "$query" X { + set DATA($X(docid)) [array get X] + } + + do_test $tn.5 { lsort [array get A] } [lsort [array get DATA]] + do_test $tn.6 { lsort [array get B] } [lsort [array get DATA]] + do_test $tn.7 { lsort [array get C] } [lsort [array get DATA]] + do_test $tn.8 { lsort [array get D] } [lsort [array get DATA]] + + if {[info exists CONTROL($tn)]} { + do_test $tn.9 { set CONTROL($tn) } [lsort [array get DATA]] + } else { + set CONTROL($tn) [lsort [array get DATA]] + } + } +} +unset -nocomplain CONTROL + +set testprefix fts3sort + +#------------------------------------------------------------------------- +# Tests for parsing the "order=asc" and "order=desc" directives. +# +foreach {tn param res} { + 1 "order=asc" {0 {}} + 2 "order=desc" {0 {}} + 3 "order=dec" {1 {unrecognized order: dec}} + 4 "order=xxx, order=asc" {0 {}} } { - - unset -nocomplain A B C D - set A_list [list] - set B_list [list] - set C_list [list] - set D_list [list] - - unset -nocomplain X - db eval "$query ORDER BY rowid ASC" X { - set A($X(docid)) [array get X] - lappend A_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY rowid DESC" X { - set B($X(docid)) [array get X] - lappend B_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY docid ASC" X { - set C($X(docid)) [array get X] - lappend C_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY docid DESC" X { - set D($X(docid)) [array get X] - lappend D_list $X(docid) - } - - do_test 1.$tn.1 { set A_list } [lsort -integer -increasing $A_list] - do_test 1.$tn.2 { set B_list } [lsort -integer -decreasing $B_list] - do_test 1.$tn.3 { set C_list } [lsort -integer -increasing $C_list] - do_test 1.$tn.4 { set D_list } [lsort -integer -decreasing $D_list] - - unset -nocomplain DATA - unset -nocomplain X - db eval "$query" X { - set DATA($X(docid)) [array get X] - } - - do_test 1.$tn.5 { lsort [array get A] } [lsort [array get DATA]] - do_test 1.$tn.6 { lsort [array get B] } [lsort [array get DATA]] - do_test 1.$tn.7 { lsort [array get C] } [lsort [array get DATA]] - do_test 1.$tn.8 { lsort [array get D] } [lsort [array get DATA]] + execsql { DROP TABLE IF EXISTS t1 } + do_catchsql_test 2.1.$tn " + CREATE VIRTUAL TABLE t1 USING fts4(a, b, $param) + " $res } +do_execsql_test 2.2 { + BEGIN; + CREATE VIRTUAL TABLE t2 USING fts4(order=desc); + INSERT INTO t2 VALUES('aa bb'); + INSERT INTO t2 VALUES('bb cc'); + INSERT INTO t2 VALUES('cc aa'); + SELECT docid FROM t2 WHERE t2 MATCH 'aa'; + END; +} {3 1} +do_execsql_test 2.3 { + SELECT docid FROM t2 WHERE t2 MATCH 'aa'; +} {3 1} + finish_test + From 76e04431ef98a94d50a64540e518df5d34c6fe0d Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Jun 2011 20:13:24 +0000 Subject: [PATCH 32/69] Remove some unreachable code. FossilOrigin-Name: 650e1a79eda5a2134a1fbd305ab1f205a57c0892 --- ext/fts3/fts3.c | 201 +----------------------------------------------- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 207 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 695d3e2d5e..5654dab214 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -447,20 +447,6 @@ static void fts3GetReverseVarint( *pVal = iVal; } -/* -** As long as *pp has not reached its end (pEnd), then do the same -** as fts3GetDeltaVarint(): read a single varint and add it to *pVal. -** But if we have reached the end of the varint, just set *pp=0 and -** leave *pVal unchanged. -*/ -static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ - if( *pp>=pEnd ){ - *pp = 0; - }else{ - fts3GetDeltaVarint(pp, pVal); - } -} - /* ** The xDisconnect() virtual table method. */ @@ -1912,188 +1898,6 @@ static int fts3PoslistNearMerge( } } -/* -** Values that may be used as the first parameter to fts3DoclistMerge(). -*/ -#define MERGE_NOT 2 /* D + D -> D */ -#define MERGE_AND 3 /* D + D -> D */ -#define MERGE_OR 4 /* D + D -> D */ -#define MERGE_PHRASE 6 /* P + P -> D */ -#define MERGE_NEAR 8 /* P + P -> D */ - -#define MERGE_POS_OR 5 /* P + P -> P */ -#define MERGE_POS_PHRASE 7 /* P + P -> P */ -#define MERGE_POS_NEAR 9 /* P + P -> P */ - -/* -** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2 -** (size n2 bytes). The output is written to pre-allocated buffer aBuffer, -** which is guaranteed to be large enough to hold the results. The number -** of bytes written to aBuffer is stored in *pnBuffer before returning. -** -** If successful, SQLITE_OK is returned. Otherwise, if a malloc error -** occurs while allocating a temporary buffer as part of the merge operation, -** SQLITE_NOMEM is returned. -*/ -static int fts3DoclistMerge( - int mergetype, /* One of the MERGE_XXX constants */ - int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - char *aBuffer, /* Pre-allocated output buffer */ - int *pnBuffer, /* OUT: Bytes written to aBuffer */ - char *a1, /* Buffer containing first doclist */ - int n1, /* Size of buffer a1 */ - char *a2, /* Buffer containing second doclist */ - int n2, /* Size of buffer a2 */ - int *pnDoc /* OUT: Number of docids in output */ -){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - - char *p = aBuffer; - char *p1 = a1; - char *p2 = a2; - char *pEnd1 = &a1[n1]; - char *pEnd2 = &a2[n2]; - int nDoc = 0; - - assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR - || mergetype==MERGE_AND || mergetype==MERGE_NOT - || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE - || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR - ); - assert( mergetype==MERGE_POS_PHRASE || mergetype==MERGE_POS_NEAR ); - - if( !aBuffer ){ - *pnBuffer = 0; - return SQLITE_NOMEM; - } - - /* Read the first docid from each doclist */ - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - - switch( mergetype ){ - case MERGE_OR: - case MERGE_POS_OR: - while( p1 || p2 ){ - if( p2 && p1 && i1==i2 ){ - fts3PutDeltaVarint(&p, &iPrev, i1); - if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - }else if( !p2 || (p1 && i1isReqPos ? MERGE_POS_OR : MERGE_OR); char *aOut = 0; int nOut = 0; int i; @@ -2406,7 +2208,6 @@ static int fts3TermSelectCb( return SQLITE_NOMEM; } }else{ - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); char *aMerge = aDoclist; int nMerge = nDoclist; int iOut; @@ -3816,7 +3617,7 @@ static void fts3EvalNearMerge( char *aOut; /* Buffer in which to assemble new doclist */ int nOut; /* Size of buffer aOut in bytes */ - *pRc = fts3DoclistNearMerge(bDescIdx, MERGE_POS_NEAR, nNear, + *pRc = fts3DoclistNearMerge(bDescIdx, nNear, pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll, pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll, &aOut, &nOut diff --git a/manifest b/manifest index 7bf78b4a1d..34f7d97ba5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\s"order=DESC"\sand\s"order=ASC"\sparameters\sin\sFTS4\s"CREATE\sVIRTUAL\sTABLE"\sstatements.\sTables\screated\swith\s"order=DESC"\sstore\sall\sdoclists\sin\sdescending\sorder,\swhich\sallows\soptimizations\snormally\sapplied\sto\s"ORDER\sBY\sdocid\sASC"\squeries\sto\sbe\sused\swith\s"ORDER\sBY\sdocid\sDESC"\squeries\sinstead. -D 2011-06-04T20:04:35.492 +C Remove\ssome\sunreachable\scode. +D 2011-06-04T20:13:24.690 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 432c902c697850cbc5ffc2be8a945ac7ac3328d7 +F ext/fts3/fts3.c 31f73cae1d21daba547c471a0a78d31e7516a29e F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a4c7e2820824e82580730c36f85aede2efa66754 -R 2bf28954217e137f11160d52b346ca5f +P f6a0193f5a32603eb48bddc6297042dbd2ffe96e +R a65937620eeff7552523045d1ea22bb7 U dan -Z b16ce48fbf2f9ba7236a3b596adb6f59 +Z 582e1f65a0e245127649a23c8e5f1248 diff --git a/manifest.uuid b/manifest.uuid index 2d5316d141..a27947ef64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f6a0193f5a32603eb48bddc6297042dbd2ffe96e \ No newline at end of file +650e1a79eda5a2134a1fbd305ab1f205a57c0892 \ No newline at end of file From 9aab071780a76e670d20a0ce58dac6cef44bbdcb Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 6 Jun 2011 06:55:38 +0000 Subject: [PATCH 33/69] Clean up the code for processing FTS4 options a bit. FossilOrigin-Name: 0425138a2365d23b07d88fda2b1f458f112f389d --- ext/fts3/fts3.c | 113 ++++++++++++++++++++++++--------------------- manifest | 14 +++--- manifest.uuid | 2 +- test/fts3sort.test | 3 +- 4 files changed, 70 insertions(+), 62 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 5654dab214..317b2b5625 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -909,30 +909,12 @@ static int fts3InitVtab( struct Fts3Index *aIndex; /* Array of indexes for this table */ struct Fts3Index *aFree = 0; /* Free this before returning */ + /* The results of parsing supported FTS4 key=value options: */ int bNoDocsize = 0; /* True to omit %_docsize table */ int bDescIdx = 0; /* True to store descending indexes */ - - char *zMatchinfo = 0; /* Prefix parameter value (or NULL) */ char *zPrefix = 0; /* Prefix parameter value (or NULL) */ char *zCompress = 0; /* compress=? parameter (or NULL) */ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ - char *zOrder = 0; /* order=? parameter (or NULL) */ - struct Fts4Option { - const char *zOpt; - int nOpt; - char **pzVar; - } aFts4Opt[] = { - { "matchinfo", 9, 0 }, - { "prefix", 6, 0 }, - { "compress", 8, 0 }, - { "uncompress", 10, 0 }, - { "order", 5, 0 } - }; - aFts4Opt[0].pzVar = &zMatchinfo; - aFts4Opt[1].pzVar = &zPrefix; - aFts4Opt[2].pzVar = &zCompress; - aFts4Opt[3].pzVar = &zUncompress; - aFts4Opt[4].pzVar = &zOrder; assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) @@ -973,26 +955,71 @@ static int fts3InitVtab( /* Check if it is an FTS4 special argument. */ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + char **pzVar; + } aFts4Opt[] = { + { "matchinfo", 9, 0 }, /* 0 -> MATCHINFO */ + { "prefix", 6, 0 }, /* 1 -> PREFIX */ + { "compress", 8, 0 }, /* 2 -> COMPRESS */ + { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ + { "order", 5, 0 } /* 4 -> ORDER */ + }; + int iOpt; if( !zVal ){ rc = SQLITE_NOMEM; }else{ for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ break; } } - if( zVal ){ + if( iOpt==SizeofArray(aFts4Opt) ){ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); rc = SQLITE_ERROR; - sqlite3_free(zVal); + }else{ + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) + ){ + *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + } } + sqlite3_free(zVal); } } @@ -1011,26 +1038,6 @@ static int fts3InitVtab( nCol = 1; } - if( zMatchinfo ){ - if( strlen(zMatchinfo)!=4 || sqlite3_strnicmp(zMatchinfo, "fts3", 4) ){ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zMatchinfo); - rc = SQLITE_ERROR; - goto fts3_init_out; - } - bNoDocsize = 1; - } - - if( zOrder ){ - if( (strlen(zOrder)!=3 || sqlite3_strnicmp(zOrder, "asc", 3)) - && (strlen(zOrder)!=4 || sqlite3_strnicmp(zOrder, "desc", 3)) - ){ - *pzErr = sqlite3_mprintf("unrecognized order: %s", zOrder); - rc = SQLITE_ERROR; - goto fts3_init_out; - } - bDescIdx = (zOrder[0]=='d' || zOrder[0]=='D'); - } - if( pTokenizer==0 ){ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); if( rc!=SQLITE_OK ) goto fts3_init_out; @@ -1042,8 +1049,7 @@ static int fts3InitVtab( assert( zPrefix ); *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); } - if( rc ) goto fts3_init_out; - + if( rc!=SQLITE_OK ) goto fts3_init_out; /* Allocate and populate the Fts3Table structure. */ nByte = sizeof(Fts3Table) + /* Fts3Table */ @@ -1130,8 +1136,6 @@ fts3_init_out: sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); - sqlite3_free(zOrder); - sqlite3_free(zMatchinfo); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ @@ -4187,7 +4191,10 @@ char *sqlite3Fts3EvalPhrasePoslist( /* ** Free all components of the Fts3Phrase structure that were allocated by -** the eval module. +** the eval module. Specifically, this means to free: +** +** * the contents of pPhrase->doclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. */ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ diff --git a/manifest b/manifest index 34f7d97ba5..3294d4e1c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sunreachable\scode. -D 2011-06-04T20:13:24.690 +C Clean\sup\sthe\scode\sfor\sprocessing\sFTS4\soptions\sa\sbit. +D 2011-06-06T06:55:38.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 31f73cae1d21daba547c471a0a78d31e7516a29e +F ext/fts3/fts3.c ea03065c8fbffc7258777fd9e1bf1c40404ce14a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f @@ -478,7 +478,7 @@ F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd -F test/fts3sort.test c599f19e9452b5735c41889ea5a6da996f0ebc7c +F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 F test/fts4aa.test eadf85621c0a113d4c7ad3ccbf8441130e007b8f F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P f6a0193f5a32603eb48bddc6297042dbd2ffe96e -R a65937620eeff7552523045d1ea22bb7 +P 650e1a79eda5a2134a1fbd305ab1f205a57c0892 +R d2808d37d9a0cf28002d639dd1fa255c U dan -Z 582e1f65a0e245127649a23c8e5f1248 +Z 406d2ac578f3de4e57d0cc24bb1a90c8 diff --git a/manifest.uuid b/manifest.uuid index a27947ef64..f78050d023 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -650e1a79eda5a2134a1fbd305ab1f205a57c0892 \ No newline at end of file +0425138a2365d23b07d88fda2b1f458f112f389d \ No newline at end of file diff --git a/test/fts3sort.test b/test/fts3sort.test index d2edbc2ade..ce5b5df76b 100644 --- a/test/fts3sort.test +++ b/test/fts3sort.test @@ -136,7 +136,8 @@ foreach {tn param res} { 1 "order=asc" {0 {}} 2 "order=desc" {0 {}} 3 "order=dec" {1 {unrecognized order: dec}} - 4 "order=xxx, order=asc" {0 {}} + 4 "order=xxx, order=asc" {1 {unrecognized order: xxx}} + 5 "order=desc, order=asc" {0 {}} } { execsql { DROP TABLE IF EXISTS t1 } do_catchsql_test 2.1.$tn " From 2ce71b4e7a11d23d9267f4c3e2714adde454ddcd Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 Jun 2011 13:38:11 +0000 Subject: [PATCH 34/69] Add assert() statements to verify that u16 pointers associated with the enhancement in [897f56a158] are always 2-byte aligned. FossilOrigin-Name: 98ccfa930e8e5f10808a518e3e22e85e8a8a65c2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index acedbbc719..9f7ebae4ea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovement\sto\sthe\sbtree\ssearch\sroutine. -D 2011-06-04T01:43:53.858 +C Add\sassert()\sstatements\sto\sverify\sthat\su16\spointers\sassociated\swith\nthe\senhancement\sin\s[897f56a158]\sare\salways\s2-byte\saligned. +D 2011-06-06T13:38:11.704 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,7 +122,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c c19d7b35c8325b302d88d73e834642a7a96ee003 +F src/btree.c 8c46f0ab69ad9549c75a3a91fed87abdaa743e2f F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 5a428625d21ad409514afb40ad083bee25dd957a @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 897f56a158ebe62758c9998e4941ae046c75fb99 -R 84139bde7a129a9544042a4a85d415c4 +P 65db822f200bafe9abe59b33b17b2c643c17c5e8 +R 6f20bbc84376df4d202ff7744a749f24 U drh -Z 4316427fea8d5aeb84c01ffb6d821ecd +Z b952c48c970134cec62f175ed76931db diff --git a/manifest.uuid b/manifest.uuid index e0e3846b64..4a6c7d51ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65db822f200bafe9abe59b33b17b2c643c17c5e8 \ No newline at end of file +98ccfa930e8e5f10808a518e3e22e85e8a8a65c2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 640d48b452..3d7162dbb3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5421,6 +5421,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ return; } endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2]; + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ while( ptrendPtr ){ *(u16*)ptr = *(u16*)&ptr[-2]; ptr -= 2; From 5289b01524c329af93ae44dfb70c8303ed553203 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 6 Jun 2011 14:51:50 +0000 Subject: [PATCH 35/69] Modify fts3rnd.test to run tests for both "ORDER BY docid ASC" and "ORDER BY docid DESC" with both order=ASC and order=DESC FTS tables. Fixes for some bugs found. FossilOrigin-Name: 89f2f482e077241ac29a58eadf44a72a9c01f98c --- ext/fts3/fts3.c | 46 +++++++++++++++++++++++----------- ext/fts3/fts3_write.c | 14 +++++++---- manifest | 16 ++++++------ manifest.uuid | 2 +- test/fts3rnd.test | 58 ++++++++++++++++++++++++++++++------------- 5 files changed, 91 insertions(+), 45 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 317b2b5625..18d7e79b78 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1144,6 +1144,7 @@ fts3_init_out: pTokenizer->pModule->xDestroy(pTokenizer); } }else{ + assert( p->pSegments==0 ); *ppVTab = &p->base; } return rc; @@ -1245,6 +1246,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } } + assert( p->pSegments==0 ); return SQLITE_OK; } @@ -1280,6 +1282,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3_free(pCsr->aMatchinfo); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_free(pCsr); return SQLITE_OK; } @@ -2564,6 +2567,7 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ }else{ rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -2701,7 +2705,7 @@ static int fts3ColumnMethod( sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; Fts3Table *p = (Fts3Table *)pCursor->pVtab; @@ -2712,21 +2716,20 @@ static int fts3ColumnMethod( /* This call is a request for the "docid" column. Since "docid" is an ** alias for "rowid", use the xRowid() method to obtain the value. */ - sqlite3_int64 iRowid; - rc = fts3RowidMethod(pCursor, &iRowid); - sqlite3_result_int64(pContext, iRowid); + sqlite3_result_int64(pContext, pCsr->iPrevId); }else if( iCol==p->nColumn ){ /* The extra column whose name is the same as the table. ** Return a blob which is a pointer to the cursor. */ sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - rc = SQLITE_OK; }else{ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -2760,6 +2763,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ static int fts3BeginMethod(sqlite3_vtab *pVtab){ UNUSED_PARAMETER(pVtab); TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + assert( p->pSegments==0 ); assert( p->nPendingData==0 ); assert( p->inTransaction!=1 ); TESTONLY( p->inTransaction = 1 ); @@ -2777,6 +2781,7 @@ static int fts3CommitMethod(sqlite3_vtab *pVtab){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); assert( p->nPendingData==0 ); assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); TESTONLY( p->inTransaction = 0 ); TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; @@ -3047,7 +3052,7 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ assert( p->inTransaction ); assert( p->mxSavepoint < iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); - return sqlite3Fts3PendingTermsFlush(p); + return fts3SyncMethod(pVtab); } static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); @@ -3459,6 +3464,10 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ return rc; } +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. +*/ void sqlite3Fts3DoclistPrev( int bDescIdx, /* True if the doclist is desc */ char *aDoclist, /* Pointer to entire doclist */ @@ -3471,6 +3480,7 @@ void sqlite3Fts3DoclistPrev( char *p = *ppIter; int iMul = (bDescIdx ? -1 : 1); + assert( nDoclist>0 ); assert( *pbEof==0 ); assert( p || *piDocid==0 ); assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); @@ -3537,7 +3547,7 @@ static int fts3EvalPhraseNext( if( rc==SQLITE_OK && !pDL->pList ){ *pbEof = 1; } - }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->aAll ){ + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof ); @@ -3879,6 +3889,7 @@ static void fts3EvalNext( int *pRc ){ if( *pRc==SQLITE_OK ){ + assert( pExpr->bEof==0 ); pExpr->bStart = 1; switch( pExpr->eType ){ @@ -3901,7 +3912,7 @@ static void fts3EvalNext( fts3EvalNext(pCsr, pRight, pRc); while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - int iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); if( iDiff==0 ) break; if( iDiff<0 ){ fts3EvalNext(pCsr, pLeft, pRc); @@ -3919,20 +3930,20 @@ static void fts3EvalNext( case FTSQUERY_OR: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; - int iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - if( iCmp==0 ){ + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ fts3EvalNext(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ fts3EvalNext(pCsr, pRight, pRc); - }else if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNext(pCsr, pLeft, pRc); }else{ + fts3EvalNext(pCsr, pLeft, pRc); fts3EvalNext(pCsr, pRight, pRc); } - + pExpr->bEof = (pLeft->bEof && pRight->bEof); iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ @@ -3972,6 +3983,12 @@ static void fts3EvalNext( fts3EvalFreeDeferredDoclist(pPhrase); *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); pExpr->iDocid = pPhrase->doclist.iDocid; +#if 0 + printf("token \"%.*s\" docid=%lld\n", + pPhrase->aToken[0].n, pPhrase->aToken[0].z, pExpr->iDocid + ); +#endif + break; } } @@ -4123,8 +4140,9 @@ int sqlite3Fts3EvalPhraseDoclist( assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); if( pExpr->bStart && !pExpr->bEof ){ pExpr->bStart = 0; - while( rc==SQLITE_OK && pExpr->bEof==0 && pExpr->iDocid!=iDocid ){ + while( rc==SQLITE_OK && (pExpr->bStart==0 || pExpr->iDocid!=iDocid) ){ fts3EvalNext(pCsr, pExpr, &rc); + assert( !pExpr->bEof ); } } } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index fd964739b2..a72556384c 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3159,7 +3159,7 @@ int sqlite3Fts3UpdateMethod( int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ - u32 *aSzIns; /* Sizes of inserted documents */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; @@ -3174,12 +3174,16 @@ int sqlite3Fts3UpdateMethod( && sqlite3_value_type(apVal[0])==SQLITE_NULL && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ - return fts3SpecialInsert(p, apVal[p->nColumn+2]); + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; } /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ) return SQLITE_NOMEM; + if( aSzIns==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } aSzDel = &aSzIns[p->nColumn+1]; memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); @@ -3229,8 +3233,7 @@ int sqlite3Fts3UpdateMethod( } } if( rc!=SQLITE_OK ){ - sqlite3_free(aSzIns); - return rc; + goto update_out; } /* If this is a DELETE or UPDATE operation, remove the old record. */ @@ -3263,6 +3266,7 @@ int sqlite3Fts3UpdateMethod( fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } + update_out: sqlite3_free(aSzIns); sqlite3Fts3SegmentsClose(p); return rc; diff --git a/manifest b/manifest index 3294d4e1c4..e6e2af3449 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\sthe\scode\sfor\sprocessing\sFTS4\soptions\sa\sbit. -D 2011-06-06T06:55:38.365 +C Modify\sfts3rnd.test\sto\srun\stests\sfor\sboth\s"ORDER\sBY\sdocid\sASC"\sand\s"ORDER\sBY\sdocid\sDESC"\swith\sboth\sorder=ASC\sand\sorder=DESC\sFTS\stables.\sFixes\sfor\ssome\sbugs\sfound. +D 2011-06-06T14:51:50.541 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c ea03065c8fbffc7258777fd9e1bf1c40404ce14a +F ext/fts3/fts3.c aafc143b59710e85a6215e8843aa2802f38397bc F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f @@ -75,7 +75,7 @@ F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c d7eded6f5ee3032b41126047cc04b6720f59e6da +F ext/fts3/fts3_write.c ed525afd524d713abe7da174d56ad935dfc26008 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -475,7 +475,7 @@ F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 -F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad +F test/fts3rnd.test d88ec3dbe52e81e410cd1a39554d15941f86333c F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 650e1a79eda5a2134a1fbd305ab1f205a57c0892 -R d2808d37d9a0cf28002d639dd1fa255c +P 0425138a2365d23b07d88fda2b1f458f112f389d +R bea111e886e30c118babca4cc7623bc5 U dan -Z 406d2ac578f3de4e57d0cc24bb1a90c8 +Z 6f031e903f656ed90101feb3a80e60ca diff --git a/manifest.uuid b/manifest.uuid index f78050d023..118c979d31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0425138a2365d23b07d88fda2b1f458f112f389d \ No newline at end of file +89f2f482e077241ac29a58eadf44a72a9c01f98c \ No newline at end of file diff --git a/test/fts3rnd.test b/test/fts3rnd.test index 0909cee614..0ad41d8ce4 100644 --- a/test/fts3rnd.test +++ b/test/fts3rnd.test @@ -162,7 +162,7 @@ proc simple_phrase {zPrefix} { # This [proc] is used to test the FTS3 matchinfo() function. # -proc simple_token_matchinfo {zToken} { +proc simple_token_matchinfo {zToken bDesc} { set nDoc(0) 0 set nDoc(1) 0 @@ -171,6 +171,8 @@ proc simple_token_matchinfo {zToken} { set nHit(1) 0 set nHit(2) 0 + set dir -inc + if {$bDesc} { set dir -dec } foreach key [array names ::t1] { set value $::t1($key) @@ -184,7 +186,7 @@ proc simple_token_matchinfo {zToken} { } set ret [list] - foreach docid [lsort -integer [array names a]] { + foreach docid [lsort -integer $dir [array names a]] { if { [lindex [lsort -integer $a($docid)] end] } { set matchinfo [list 1 3] foreach i {0 1 2} hit $a($docid) { @@ -262,17 +264,30 @@ proc mit {blob} { return $r } db func mit mit - set sqlite_fts3_enable_parentheses 1 -foreach nodesize {50 500 1000 2000} { +proc do_orderbydocid_test {tn sql res} { + uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res] + uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \ + [lsort -int -dec $res] + ] +} + +foreach {nodesize order} { + 50 DESC + 50 ASC + 500 ASC + 1000 DESC + 2000 ASC +} { catch { array unset ::t1 } + set testname "$nodesize/$order" # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. # db transaction { catchsql { DROP TABLE t1 } - execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)" + execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)" execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')" for {set i 0} {$i < 100} {incr i} { insert_row $i } } @@ -286,6 +301,8 @@ foreach nodesize {50 500 1000 2000} { set DO_MALLOC_TEST 1 set nRep 2 } + + set ::testprefix fts3rnd-1.$testname.$iTest # Delete one row, update one row and insert one row. # @@ -307,7 +324,7 @@ foreach nodesize {50 500 1000 2000} { if {0==($iTest%2)} { execsql COMMIT } if {0==($iTest%2)} { - do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok + #do_test 0 { fts3_integrity_check t1 } ok } # Pick 10 terms from the vocabulary. Check that the results of querying @@ -317,9 +334,14 @@ foreach nodesize {50 500 1000 2000} { # for {set i 0} {$i < 10} {incr i} { set term [random_term] - do_select_test fts3rnd-1.$nodesize.$iTest.1.$i { + do_select_test 1.$i { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term - } [simple_token_matchinfo $term] + ORDER BY docid ASC + } [simple_token_matchinfo $term 0] + do_select_test 1.$i { + SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term + ORDER BY docid DESC + } [simple_token_matchinfo $term 1] } # This time, use the first two characters of each term as a term prefix @@ -329,7 +351,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set prefix [string range [random_term] 0 end-1] set match "${prefix}*" - do_select_test fts3rnd-1.$nodesize.$iTest.2.$i { + do_orderbydocid_test 2.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $match] } @@ -339,7 +361,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term]] set match "\"$term\"" - do_select_test fts3rnd-1.$nodesize.$iTest.3.$i { + do_orderbydocid_test 3.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } @@ -349,7 +371,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term] [random_term]] set match "\"$term\"" - do_select_test fts3rnd-1.$nodesize.$iTest.4.$i { + do_orderbydocid_test 4.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } @@ -362,17 +384,19 @@ foreach nodesize {50 500 1000 2000} { append query "[string range [random_term] 0 end-1]*" set match "\"$query\"" - do_select_test fts3rnd-1.$nodesize.$iTest.5.$i { + do_orderbydocid_test 5.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $query] } - # A NEAR query with terms as the arguments. + # A NEAR query with terms as the arguments: + # + # ... MATCH '$term1 NEAR $term2' ... # for {set i 0} {$i < $nRep} {incr i} { set terms [list [random_term] [random_term]] set match [join $terms " NEAR "] - do_select_test fts3rnd-1.$nodesize.$iTest.6.$i { + do_orderbydocid_test 6.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms 10] } @@ -383,7 +407,7 @@ foreach nodesize {50 500 1000 2000} { set terms [list [random_term] [random_term] [random_term]] set nNear 11 set match [join $terms " NEAR/$nNear "] - do_select_test fts3rnd-1.$nodesize.$iTest.7.$i { + do_orderbydocid_test 7.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms $nNear] } @@ -399,7 +423,7 @@ foreach nodesize {50 500 1000 2000} { set term1 [random_term] set term2 [random_term] set match "$term1 $op $term2" - do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + do_orderbydocid_test $tn.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc [simple_phrase $term1] [simple_phrase $term2]] } @@ -418,7 +442,7 @@ foreach nodesize {50 500 1000 2000} { set term3 [random_term] set term4 [random_term] set match "$term1 NEAR $term2 $op $term3 NEAR $term4" - do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + do_orderbydocid_test $tn.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc \ [simple_near [list $term1 $term2] 10] \ From 55a0959a181bcd3d75a540df375b5d5e3904fb0f Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Jun 2011 18:31:14 +0000 Subject: [PATCH 36/69] Fix a comment type on the description of the Schema object. FossilOrigin-Name: 095cd9a6ec175b703ff3fcafeffb3349f21bd831 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9f7ebae4ea..c598385e18 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sassert()\sstatements\sto\sverify\sthat\su16\spointers\sassociated\swith\nthe\senhancement\sin\s[897f56a158]\sare\salways\s2-byte\saligned. -D 2011-06-06T13:38:11.704 +C Fix\sa\scomment\stype\son\sthe\sdescription\sof\sthe\sSchema\sobject. +D 2011-06-07T18:31:14.409 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 F src/sqlite.h.in 2f51e4f58b2b4626fcbd9938580e730cb5fb4985 F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h 6e58c558c57c8f44011736d5fa5295eb3130f9de +F src/sqliteInt.h d2a9f6e06b85bb72a47cfe8d45320abe9cfa44f1 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 65db822f200bafe9abe59b33b17b2c643c17c5e8 -R 6f20bbc84376df4d202ff7744a749f24 +P 98ccfa930e8e5f10808a518e3e22e85e8a8a65c2 +R eade3cd28243c0adfa4f9b0e3371ab06 U drh -Z b952c48c970134cec62f175ed76931db +Z 6c3c2f88a9e5ad93debb108bda4e78cb diff --git a/manifest.uuid b/manifest.uuid index 4a6c7d51ec..df6972f87d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98ccfa930e8e5f10808a518e3e22e85e8a8a65c2 \ No newline at end of file +095cd9a6ec175b703ff3fcafeffb3349f21bd831 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index fd86158e00..8cf8966fe0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -682,7 +682,7 @@ struct Db { ** A thread must be holding a mutex on the corresponding Btree in order ** to access Schema content. This implies that the thread must also be ** holding a mutex on the sqlite3 connection pointer that owns the Btree. -** For a TEMP Schema, on the connection mutex is required. +** For a TEMP Schema, only the connection mutex is required. */ struct Schema { int schema_cookie; /* Database schema version number for this file */ From 3eabcf5f4686047b82cf31c11c90f462cd7c346f Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Jun 2011 18:35:45 +0000 Subject: [PATCH 37/69] Have NEAR queries use incremental merging. Fix issues surrounding the deferred token optimization. FossilOrigin-Name: 9d10a6846b12a9cc8fd4fdc3affd931a27218b5a --- ext/fts3/fts3.c | 363 ++++++++++++++++++++++++++++++++-------- manifest | 18 +- manifest.uuid | 2 +- test/fts3defer2.test | 17 +- test/fts3matchinfo.test | 4 + 5 files changed, 320 insertions(+), 84 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 0ab0ecaef7..d3a49f9df6 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1069,7 +1069,6 @@ static int fts3InitVtab( p->nPendingData = 0; p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; - p->nNodeSize = 1000; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; @@ -1127,6 +1126,7 @@ static int fts3InitVtab( ** function sqlite3Fts3SegReaderCost() for details). */ fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); @@ -1862,7 +1862,19 @@ static int fts3PoslistPhraseMerge( } /* -** Merge two position-lists as required by the NEAR operator. +** Merge two position-lists as required by the NEAR operator. The argument +** position lists correspond to the left and right phrases of an expression +** like: +** +** "phrase 1" NEAR "phrase number 2" +** +** Position list *pp1 corresponds to the left-hand side of the NEAR +** expression and *pp2 to the right. As usual, the indexes in the position +** lists are the offsets of the last token in each phrase (tokens "1" and "2" +** in the example above). +** +** The output position list - written to *pp - is a copy of *pp2 with those +** entries that are not sufficiently NEAR entries in *pp1 removed. */ static int fts3PoslistNearMerge( char **pp, /* Output buffer */ @@ -1875,34 +1887,27 @@ static int fts3PoslistNearMerge( char *p1 = *pp1; char *p2 = *pp2; - if( !pp ){ - if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1; - *pp1 = p1; - *pp2 = p2; - return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1); + char *pTmp1 = aTmp; + char *pTmp2; + char *aTmp2; + int res = 1; + + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); + aTmp2 = pTmp2 = pTmp1; + *pp1 = p1; + *pp2 = p2; + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); + if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ + fts3PoslistMerge(pp, &aTmp, &aTmp2); + }else if( pTmp1!=aTmp ){ + fts3PoslistCopy(pp, &aTmp); + }else if( pTmp2!=aTmp2 ){ + fts3PoslistCopy(pp, &aTmp2); }else{ - char *pTmp1 = aTmp; - char *pTmp2; - char *aTmp2; - int res = 1; - - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); - aTmp2 = pTmp2 = pTmp1; - *pp1 = p1; - *pp2 = p2; - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); - if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ - fts3PoslistMerge(pp, &aTmp, &aTmp2); - }else if( pTmp1!=aTmp ){ - fts3PoslistCopy(pp, &aTmp); - }else if( pTmp2!=aTmp2 ){ - fts3PoslistCopy(pp, &aTmp2); - }else{ - res = 0; - } - - return res; + res = 0; } + + return res; } /* @@ -3244,6 +3249,7 @@ static void fts3EvalAllocateReaders( Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ int *pRc ){ if( pExpr && SQLITE_OK==*pRc ){ @@ -3262,8 +3268,9 @@ static void fts3EvalAllocateReaders( } } }else{ - fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pRc); - fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pRc); + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); } } } @@ -3545,6 +3552,7 @@ static int fts3EvalPhraseNext( if( p->bIncr ){ assert( p->nToken==1 ); + assert( pDL->pNextDocid==0 ); rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, &pDL->iDocid, &pDL->pList, &pDL->nList ); @@ -3602,9 +3610,6 @@ static void fts3EvalStartReaders( pExpr->bDeferred = (i==nToken); *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); }else{ - if( pExpr->eType==FTSQUERY_NEAR ){ - bOptOk = 0; - } fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); @@ -3700,14 +3705,17 @@ static void fts3EvalNearTrim(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ typedef struct Fts3TokenAndCost Fts3TokenAndCost; struct Fts3TokenAndCost { Fts3PhraseToken *pToken; + Fts3Expr *pRoot; int nOvfl; int iCol; }; static void fts3EvalTokenCosts( Fts3Cursor *pCsr, + Fts3Expr *pRoot, Fts3Expr *pExpr, Fts3TokenAndCost **ppTC, + Fts3Expr ***ppOr, int *pRc ){ if( *pRc==SQLITE_OK && pExpr ){ @@ -3716,13 +3724,19 @@ static void fts3EvalTokenCosts( int i; for(i=0; *pRc==SQLITE_OK && inToken; i++){ Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pRoot = pRoot; pTC->pToken = &pPhrase->aToken[i]; pTC->iCol = pPhrase->iColumn; *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); } - }else if( pExpr->eType==FTSQUERY_AND ){ - fts3EvalTokenCosts(pCsr, pExpr->pLeft, ppTC, pRc); - fts3EvalTokenCosts(pCsr, pExpr->pRight, ppTC, pRc); + }else if( pExpr->eType!=FTSQUERY_NOT ){ + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr; + **ppOr = pExpr; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); } } } @@ -3773,13 +3787,87 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ return SQLITE_OK; } +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, + Fts3Expr *pRoot, + Fts3TokenAndCost *aTC, + int nTC +){ + int nDocSize = 0; + int nDocEst = 0; + int rc = SQLITE_OK; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int ii; + + int nOvfl = 0; + int nTerm = 0; + + for(ii=0; iinOvfl) + ){ + pTC = &aTC[jj]; + } + } + assert( pTC ); + + /* At this point pTC points to the cheapest remaining token. */ + if( ii==0 ){ + if( pTC->nOvfl ){ + nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; + }else{ + /* TODO: Fix this so that the doclist need not be read twice. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + if( rc==SQLITE_OK ){ + nDocEst = fts3DoclistCountDocids(1, pList, nList); + } + sqlite3_free(pList); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + } + } + }else{ + if( pTC->nOvfl>=(nDocEst*nDocSize) ){ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + } + nDocEst = 1 + (nDocEst/4); + } + pTC->pToken = 0; + } + + return rc; +} + int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; int nToken = 0; + int nOr = 0; /* Allocate a MultiSegReader for each token in the expression. */ - fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &rc); + fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: ** This call will eventually also be responsible for determining which @@ -3799,23 +3887,35 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ ** call. This is known as a "deferred" token. */ - /* If bOptOk is true, check if there are any tokens that can be - ** deferred (strategy 3). */ + /* If bOptOk is true, check if there are any tokens that should be deferred. + */ if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ Fts3TokenAndCost *aTC; - aTC = (Fts3TokenAndCost *)sqlite3_malloc(sizeof(Fts3TokenAndCost) * nToken); + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr + ); + apOr = (Fts3Expr **)&aTC[nToken]; + if( !aTC ){ rc = SQLITE_NOMEM; }else{ int ii; - int nDocEst = 0; int nDocSize; Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; - rc = fts3EvalAverageDocsize(pCsr, &nDocSize); - fts3EvalTokenCosts(pCsr, pExpr, &pTC, &rc); + fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); nToken = pTC-aTC; + nOr = ppOr-apOr; + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iinOvfl ){ @@ -3857,15 +3958,13 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ } pTC->pToken = 0; } +#endif + sqlite3_free(aTC); } } fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc); - - /* Fix the results of NEAR expressions. */ - fts3EvalNearTrim(pCsr, pExpr, &rc); - return rc; } @@ -3878,6 +3977,98 @@ static void fts3EvalFreeDeferredDoclist(Fts3Phrase *pPhrase){ } } +static int fts3EvalNearTrim2( + int nNear, + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + char *p2; + char *pOut; + int res; + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + pPhrase->doclist.nList = pOut - pPhrase->doclist.pList; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + + return res; +} + +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && pExpr->bEof==0 + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + int nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear = p->pParent->nNear; + Fts3Phrase *pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + + return res; +} + /* ** This macro is used by the fts3EvalNext() function. The two arguments are ** 64-bit docid values. If the current query is "ORDER BY docid ASC", then @@ -3901,7 +4092,6 @@ static void fts3EvalNext( case FTSQUERY_AND: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; - assert( !pLeft->bDeferred || !pRight->bDeferred ); if( pLeft->bDeferred ){ fts3EvalNext(pCsr, pRight, pRc); @@ -3924,7 +4114,7 @@ static void fts3EvalNext( fts3EvalNext(pCsr, pRight, pRc); } } - + pExpr->iDocid = pLeft->iDocid; pExpr->bEof = (pLeft->bEof || pRight->bEof); } @@ -3987,12 +4177,6 @@ static void fts3EvalNext( fts3EvalFreeDeferredDoclist(pPhrase); *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); pExpr->iDocid = pPhrase->doclist.iDocid; -#if 0 - printf("token \"%.*s\" docid=%lld\n", - pPhrase->aToken[0].n, pPhrase->aToken[0].z, pExpr->iDocid - ); -#endif - break; } } @@ -4000,7 +4184,7 @@ static void fts3EvalNext( } static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ - int bHit = 0; + int bHit = 1; if( *pRc==SQLITE_OK ){ switch( pExpr->eType ){ case FTSQUERY_NEAR: @@ -4008,15 +4192,40 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ bHit = ( fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) ); + + /* If this is a NEAR node and the NEAR expression does not match + ** any rows, zero the doclist for all phrases involved in the NEAR. + ** This is because the snippet(), offsets() and matchinfo() functions + ** are not supposed to recognize any instances of phrases that are + ** part of unmatched NEAR queries. For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( pExpr->eType==FTSQUERY_NEAR && bHit==0 ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + p->pRight->pPhrase->doclist.pList = 0; + } + p->pPhrase->doclist.pList = 0; + } + break; - case FTSQUERY_OR: - bHit = ( - fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) - || fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) - ); + case FTSQUERY_OR: { + int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; break; + } case FTSQUERY_NOT: bHit = ( @@ -4026,11 +4235,15 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ break; default: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalFreeDeferredDoclist(pPhrase); - *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); - bHit = (pPhrase->doclist.pList!=0); - pExpr->iDocid = pCsr->iPrevId; + if( pCsr->pDeferred ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalFreeDeferredDoclist(pPhrase); + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else{ + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); + } break; } } @@ -4052,11 +4265,12 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; int bMiss = 0; - if( rc==SQLITE_OK && pCsr->pDeferred ){ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - sqlite3Fts3FreeDeferredDoclists(pCsr); - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + if( rc==SQLITE_OK ){ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } } bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); sqlite3Fts3FreeDeferredDoclists(pCsr); @@ -4151,6 +4365,13 @@ int sqlite3Fts3EvalPhraseDoclist( } } + if( rc==SQLITE_OK + && pExpr->pParent + && pExpr->pParent->eType==FTSQUERY_NEAR + ){ + + } + *pnList = pPhrase->doclist.nAll; *ppList = pPhrase->doclist.aAll; return rc; diff --git a/manifest b/manifest index 8000db4360..a97b87491f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sfts3-prefix-search\sbranch. -D 2011-06-06T18:14:25.721 +C Have\sNEAR\squeries\suse\sincremental\smerging.\sFix\sissues\ssurrounding\sthe\sdeferred\stoken\soptimization. +D 2011-06-07T18:35:45.780 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c d0970d9fa0a63ac470007ac5e14645f5b874b70f +F ext/fts3/fts3.c 9d2d2cab4d64f0769046d88b6740c6e1f229d1e3 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -464,14 +464,14 @@ F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35 -F test/fts3defer2.test 288bef6de15557319b8c12d476ebdc83688ef96c +F test/fts3defer2.test ad5bd3ae616fb719eb06b0a73eb5c8fb1c411989 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b -F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba +F test/fts3matchinfo.test f424597b6843659ecbc2009e8823380233ebf375 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 @@ -943,7 +943,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 89f2f482e077241ac29a58eadf44a72a9c01f98c 98ccfa930e8e5f10808a518e3e22e85e8a8a65c2 -R b612d83743a894702aacbc42c433c9e9 -U drh -Z c6414e08a953b76b4b33ed29bc763b72 +P 567dd84359218245d4e6887547e2a48881f2c8e0 +R 020efe4a51ef4472e0e5c3f4175d0de6 +U dan +Z 740d1ddba83232619fca71041707ab60 diff --git a/manifest.uuid b/manifest.uuid index 577710ac6f..4da09088b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -567dd84359218245d4e6887547e2a48881f2c8e0 \ No newline at end of file +9d10a6846b12a9cc8fd4fdc3affd931a27218b5a \ No newline at end of file diff --git a/test/fts3defer2.test b/test/fts3defer2.test index 8441803300..b7bf923550 100644 --- a/test/fts3defer2.test +++ b/test/fts3defer2.test @@ -48,17 +48,22 @@ do_execsql_test 1.1.4 { UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000; } {2} +do_execsql_test 1.2.0 { + SELECT content FROM t1 WHERE t1 MATCH 'f (e a)'; +} {{a b c d e f a x y}} + do_execsql_test 1.2.1 { SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } {{a b c d e f a x y}} +breakpoint do_execsql_test 1.2.2 { SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } [list \ {a b c d [e] [f] [a] x y} \ {0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 8 8 1 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 1 8 8 8 5001 9] ] do_execsql_test 1.2.3 { @@ -67,7 +72,7 @@ do_execsql_test 1.2.3 { } [list \ {[a] b c d [e] [f] [a] x y} \ {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 8 8 2 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 2 8 8 8 5001 9] ] do_execsql_test 1.3.1 { DROP TABLE t1 } @@ -99,8 +104,14 @@ foreach {tn sql} { [list 2 1 1 54 54 1 3 3 54 372 7] \ ] - set sqlite_fts3_enable_parentheses 1 do_execsql_test 2.2.$tn.2 { + SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g z'; + } [list \ + [list 1 2 2 1 54 54] \ + ] + + set sqlite_fts3_enable_parentheses 1 + do_execsql_test 2.2.$tn.3 { SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)'; } [list \ [list 1 2 2 1 2 2 1 54 54] \ diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index 8f194e72cb..e26bcf1052 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -68,7 +68,11 @@ do_execsql_test 3.1 { do_execsql_test 3.2 { CREATE VIRTUAL TABLE xx USING FTS4; +} +do_execsql_test 3.3 { SELECT * FROM xx WHERE xx MATCH 'abc'; +} +do_execsql_test 3.4 { SELECT * FROM xx WHERE xx MATCH 'a b c'; } From abf2545ed9c058bd62502b858adbdddcdabef446 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 8 Jun 2011 18:39:07 +0000 Subject: [PATCH 38/69] Fix various issues to do with deferred tokens, NEAR expressions and matchinfo(). FossilOrigin-Name: 3972a787df5ec253b99b148385655e7b68d851fa --- ext/fts3/fts3.c | 525 +++++++++++++++++----------------------- ext/fts3/fts3Int.h | 25 +- ext/fts3/fts3_expr.c | 1 + ext/fts3/fts3_snippet.c | 49 +--- ext/fts3/fts3_write.c | 105 -------- manifest | 22 +- manifest.uuid | 2 +- test/fts3matchinfo.test | 10 +- 8 files changed, 265 insertions(+), 474 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index d3a49f9df6..5536a684ae 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1122,9 +1122,7 @@ static int fts3InitVtab( } /* Figure out the page-size for the database. This is required in order to - ** estimate the cost of loading large doclists from the database (see - ** function sqlite3Fts3SegReaderCost() for details). - */ + ** estimate the cost of loading large doclists from the database. */ fts3DatabasePageSize(&rc, p); p->nNodeSize = p->nPgsz-35; @@ -1965,9 +1963,9 @@ static void fts3PutDeltaVarint3( static int fts3DoclistOrMerge( int bDescIdx, /* True if arguments are desc */ - u8 *a1, int n1, /* First doclist */ - u8 *a2, int n2, /* Second doclist */ - u8 **paOut, int *pnOut /* OUT: Malloc'd doclist */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; @@ -1977,7 +1975,6 @@ static int fts3DoclistOrMerge( char *p1 = a1; char *p2 = a2; char *p; - int nOut; char *aOut; int bFirstOut = 0; @@ -2016,8 +2013,8 @@ static int fts3DoclistOrMerge( static void fts3DoclistPhraseMerge( int bDescIdx, /* True if arguments are desc */ int nDist, /* Distance from left to right (1=adjacent) */ - u8 *aLeft, int nLeft, /* Left doclist */ - u8 *aRight, int *pnRight /* IN/OUT: Right/output doclist */ + char *aLeft, int nLeft, /* Left doclist */ + char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; @@ -2063,83 +2060,6 @@ static void fts3DoclistPhraseMerge( *pnRight = p - aOut; } -/* -** This function merges two doclists according to the requirements of a -** NEAR operator. -*/ -static int fts3DoclistNearMerge( - int bDescIdx, - int nNear, /* Parameter to NEAR operator */ - int nTokenLeft, /* Number of tokens in LHS phrase arg */ - char *aLeft, /* Doclist for LHS (incl. positions) */ - int nLeft, /* Size of LHS doclist in bytes */ - int nTokenRight, /* As nTokenLeft */ - char *aRight, /* As aLeft */ - int nRight, /* As nRight */ - char **paOut, /* OUT: Results of merge (malloced) */ - int *pnOut /* OUT: Sized of output buffer */ -){ - char *aOut; /* Buffer to write output doclist to */ - char *aTmp; /* Temp buffer used by PoslistNearMerge() */ - - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - int bFirstOut = 0; - - char *pEnd1 = &aLeft[nLeft]; - char *pEnd2 = &aRight[nRight]; - char *p1 = aLeft; - char *p2 = aRight; - char *p; - - int nParam1 = nNear+nTokenRight; - int nParam2 = nNear+nTokenLeft; - - p = aOut = sqlite3_malloc(nLeft+nRight+1); - aTmp = sqlite3_malloc(2*(nLeft+nRight+1)); - if( !aOut || !aTmp ){ - sqlite3_free(aOut); - sqlite3_free(aTmp); - *paOut = 0; - *pnOut = 0; - return SQLITE_NOMEM; - } - - fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); - - while( p1 && p2 ){ - sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); - if( iDiff==0 ){ - char *pSave = p; - sqlite3_int64 iPrevSave = iPrev; - int bFirstOutSave = bFirstOut; - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); - if( !fts3PoslistNearMerge(&p, aTmp, nParam1, nParam2, &p1, &p2) ){ - p = pSave; - iPrev = iPrevSave; - bFirstOut = bFirstOutSave; - } - - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); - }else if( iDiff<0 ){ - fts3PoslistCopy(0, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - }else{ - fts3PoslistCopy(0, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); - } - } - - sqlite3_free(aTmp); - *paOut = aOut; - *pnOut = p - aOut; - return SQLITE_OK; -} - - /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single @@ -2166,7 +2086,7 @@ static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ pTS->aaOutput[i] = 0; }else{ int nNew; - u8 *aNew; + char *aNew; int rc = fts3DoclistOrMerge(p->bDescIdx, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew @@ -2231,7 +2151,7 @@ static int fts3TermSelectCb( pTS->anOutput[iOut] = nMerge; break; }else{ - u8 *aNew; + char *aNew; int nNew; int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, @@ -2403,7 +2323,6 @@ int sqlite3Fts3TermSegReaderCursor( pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); if( pSegcsr ){ int i; - int nCost = 0; int bFound = 0; /* True once an index has been found */ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; @@ -2436,10 +2355,6 @@ int sqlite3Fts3TermSegReaderCursor( ); pSegcsr->bLookup = !isPrefix; } - for(i=0; rc==SQLITE_OK && inSegment; i++){ - rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); - } - pSegcsr->nCost = nCost; } *ppSegcsr = pSegcsr; @@ -3053,11 +2968,10 @@ static int fts3RenameMethod( } static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); - assert( p->inTransaction ); - assert( p->mxSavepoint < iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint ); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); return fts3SyncMethod(pVtab); } static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ @@ -3328,7 +3242,6 @@ static int fts3EvalPhraseLoad( } static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int iToken; int rc = SQLITE_OK; @@ -3450,12 +3363,10 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ int rc; - Fts3Doclist *pList = &p->doclist; Fts3PhraseToken *pFirst = &p->aToken[0]; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - assert( pList->aAll==0 ); - + assert( p->doclist.aAll==0 ); if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup ){ @@ -3565,14 +3476,15 @@ static int fts3EvalPhraseNext( ); pDL->pList = pDL->pNextDocid; }else{ - char *pIter; + char *pIter; /* Used to iterate through aAll */ + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ if( pDL->pNextDocid ){ pIter = pDL->pNextDocid; }else{ pIter = pDL->aAll; } - if( pIter>=&pDL->aAll[pDL->nAll] ){ + if( pIter>=pEnd ){ /* We have already reached the end of this doclist. EOF. */ *pbEof = 1; }else{ @@ -3586,7 +3498,17 @@ static int fts3EvalPhraseNext( pDL->pList = pIter; fts3PoslistCopy(0, &pIter); pDL->nList = (pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim2(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim2(). */ + while( pIterpNextDocid = pIter; + assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] ); *pbEof = 0; } } @@ -3617,90 +3539,6 @@ static void fts3EvalStartReaders( } } -static void fts3EvalNearMerge( - int bDescIdx, - Fts3Expr *p1, - Fts3Expr *p2, - int nNear, - int *pRc -){ - if( *pRc==SQLITE_OK ){ - int rc; /* Return code */ - Fts3Phrase *pLeft = p1->pPhrase; - Fts3Phrase *pRight = p2->pPhrase; - - assert( p2->eType==FTSQUERY_PHRASE && pLeft ); - assert( p2->eType==FTSQUERY_PHRASE && pRight ); - - if( pLeft->doclist.aAll==0 ){ - sqlite3_free(pRight->doclist.aAll); - pRight->doclist.aAll = 0; - pRight->doclist.nAll = 0; - }else if( pRight->doclist.aAll ){ - char *aOut; /* Buffer in which to assemble new doclist */ - int nOut; /* Size of buffer aOut in bytes */ - - *pRc = fts3DoclistNearMerge(bDescIdx, nNear, - pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll, - pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll, - &aOut, &nOut - ); - sqlite3_free(pRight->doclist.aAll); - pRight->doclist.aAll = aOut; - pRight->doclist.nAll = nOut; - } - } -} - -static void fts3EvalNearTrim(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ - - if( pExpr && SQLITE_OK==*pRc ){ - if( pExpr->eType==FTSQUERY_NEAR ){ - Fts3Expr *pLeft = pExpr->pLeft; - int nPhrase = 2; - Fts3Expr **aPhrase; - - assert( pLeft ); - assert( pExpr->pRight ); - assert( pExpr->pRight->eType==FTSQUERY_PHRASE ); - - while( pLeft->eType!=FTSQUERY_PHRASE ){ - assert( pLeft->eType==FTSQUERY_NEAR ); - assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); - pLeft = pLeft->pLeft; - nPhrase++; - } - - aPhrase = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nPhrase); - if( !aPhrase ){ - *pRc = SQLITE_NOMEM; - }else{ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - int i = 1; - aPhrase[0] = pLeft; - do { - pLeft = pLeft->pParent; - aPhrase[i++] = pLeft->pRight; - }while( pLeft!=pExpr ); - - for(i=0; i<(nPhrase-1); i++){ - int nNear = aPhrase[i+1]->pParent->nNear; - fts3EvalNearMerge(p->bDescIdx, aPhrase[i], aPhrase[i+1], nNear, pRc); - } - for(i=nPhrase-2; i>=0; i--){ - int nNear = aPhrase[i+1]->pParent->nNear; - fts3EvalNearMerge(p->bDescIdx, aPhrase[i+1], aPhrase[i], nNear, pRc); - } - - sqlite3_free(aPhrase); - } - - }else{ - fts3EvalNearTrim(pCsr, pExpr->pLeft, pRc); - fts3EvalNearTrim(pCsr, pExpr->pRight, pRc); - } - } -} typedef struct Fts3TokenAndCost Fts3TokenAndCost; struct Fts3TokenAndCost { @@ -3777,6 +3615,7 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ return SQLITE_CORRUPT_VTAB; } + pCsr->nDoc = nDoc; pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); assert( pCsr->nRowAvg>0 ); rc = sqlite3_reset(pStmt); @@ -3902,7 +3741,6 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ rc = SQLITE_NOMEM; }else{ int ii; - int nDocSize; Fts3TokenAndCost *pTC = aTC; Fts3Expr **ppOr = apOr; @@ -3910,55 +3748,12 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ nToken = pTC-aTC; nOr = ppOr-apOr; - rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); - for(ii=0; rc==SQLITE_OK && iinOvfl) ){ - pTC = &aTC[jj]; - } + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iinOvfl ){ - nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; - }else{ - /* TODO: Fix this so that the doclist need not be read twice. */ - Fts3PhraseToken *pToken = pTC->pToken; - int nList = 0; - char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); - if( rc==SQLITE_OK ){ - nDocEst = fts3DoclistCountDocids(1, pList, nList); - } - sqlite3_free(pList); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3TermSegReaderCursor(pCsr, - pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr - ); - } - } - }else{ - if( pTC->nOvfl>=(nDocEst*nDocSize) ){ - Fts3PhraseToken *pToken = pTC->pToken; - rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); - fts3SegReaderCursorFree(pToken->pSegcsr); - pToken->pSegcsr = 0; - } - nDocEst = 1 + (nDocEst/4); - } - pTC->pToken = 0; } -#endif sqlite3_free(aTC); } @@ -3986,6 +3781,7 @@ static int fts3EvalNearTrim2( ){ int nParam1 = nNear + pPhrase->nToken; int nParam2 = nNear + *pnToken; + int nNew; char *p2; char *pOut; int res; @@ -3994,9 +3790,15 @@ static int fts3EvalNearTrim2( res = fts3PoslistNearMerge( &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 ); - pPhrase->doclist.nList = pOut - pPhrase->doclist.pList; - *paPoslist = pPhrase->doclist.pList; - *pnToken = pPhrase->nToken; + if( res ){ + nNew = (pOut - pPhrase->doclist.pList) - 1; + assert( pPhrase->doclist.pList[nNew]=='\0' ); + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } return res; } @@ -4305,75 +4107,202 @@ int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ return rc; } -/* -** Return a pointer to the entire doclist, including positions, associated -** with the phrase passed as the second argument. It is illegal to call -** this function if the phrase consists entirely of deferred tokens. -** -** TODO: This function is only used by the code for the matchinfo('x') -** auxiliary function - to obtain the following two values: -** -** 1. The total number of times the phrase appears in each column in all -** rows in the FTS table. -** -** 2. For each column, the total number of rows in the FTS table for which -** the phrase appears at least once in the column. -** -** It would be better if there was an sqlite3Fts3EvalXXX() function -** specifically to retrieve these values. If that were done, the concept -** of which tokens are deferred or incremental would be entirely encapsulated -** within the sqlite3Fts3EvalXXX()/fts3EvalXXX() functions in this file. -*/ -int sqlite3Fts3EvalPhraseDoclist( - Fts3Cursor *pCsr, /* FTS3 cursor object */ - Fts3Expr *pExpr, /* Phrase to return doclist for */ - const char **ppList, /* OUT: Buffer containing doclist */ - int *pnList /* OUT: Size of returned buffer, in bytes */ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc ){ - int rc = SQLITE_OK; - Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; - /* It is illegal to call this function if the phrase is entirely deferred - ** (it may contain some deferred tokens, but must also contain at least - ** one token for which the doclist may be read from the full-text index). - */ - assert( !pExpr->bDeferred ); + if( pPhrase ){ + fts3EvalFreeDeferredDoclist(pPhrase); + if( pPhrase->bIncr ){ + sqlite3Fts3EvalPhraseCleanup(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + *pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0); + }else{ + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + } + } - if( pPhrase->bIncr ){ - /* This phrase was being loaded from disk incrementally. But the - ** matchinfo() function requires that the entire doclist be loaded into - ** memory. This block loads the doclist into memory and modifies the - ** Fts3Phrase structure so that it does not use the incremental strategy. - */ - TESTONLY( int bEof = pExpr->bEof; ) - TESTONLY( int bStart = pExpr->bStart; ) - sqlite3_int64 iDocid = pExpr->iDocid; - - sqlite3Fts3EvalPhraseCleanup(pPhrase); pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; - rc = sqlite3Fts3EvalStart(pCsr, pExpr, 0); - assert( pExpr->bEof==bEof ); - assert( pExpr->bStart==bStart ); - assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); - if( pExpr->bStart && !pExpr->bEof ){ - pExpr->bStart = 0; - while( rc==SQLITE_OK && (pExpr->bStart==0 || pExpr->iDocid!=iDocid) ){ - fts3EvalNext(pCsr, pExpr, &rc); - assert( !pExpr->bEof ); + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +static void fts3EvalUpdateCounts( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + assert( *p ); + while( 1 ){ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += sqlite3Fts3GetVarint32(p, &iCol); + } + } + + fts3EvalUpdateCounts(pCsr, pExpr->pLeft, pRc); + fts3EvalUpdateCounts(pCsr, pExpr->pRight, pRc); + } +} + +static int fts3EvalNearStats( + Fts3Cursor *pCsr, + Fts3Expr *pExpr +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } + + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNext(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && fts3EvalLoadDeferred(pCsr, &rc) + ); + + if( pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pCsr, pRoot, &rc); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + fts3EvalRestart(pCsr, pRoot, &rc); + while( pRoot->iDocidbEof==0 ); + } + fts3EvalLoadDeferred(pCsr, &rc); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pCsr->nDoc; + aiOut[iCol*3 + 2] = pCsr->nDoc; + } + }else{ + rc = fts3EvalNearStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; } } } - if( rc==SQLITE_OK - && pExpr->pParent - && pExpr->pParent->eType==FTSQUERY_NEAR - ){ - - } - - *pnList = pPhrase->doclist.nAll; - *ppList = pPhrase->doclist.aAll; return rc; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index c6329c30d0..80604d6fa3 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -240,6 +240,7 @@ struct Fts3Cursor { u8 bDesc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ + int nDoc; /* Documents in table */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ @@ -323,9 +324,17 @@ struct Fts3Phrase { ** "Length" field found in doclists stored on disk is omitted from this ** buffer. ** -** Variable pCurrent always points to the start of a docid field within -** aDoclist. Since the doclist is usually scanned in docid order, this can -** be used to accelerate seeking to the required docid within the doclist. +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. */ struct Fts3Expr { int eType; /* One of the FTSQUERY_XXX values defined below */ @@ -340,6 +349,8 @@ struct Fts3Expr { u8 bEof; /* True this expression is at EOF already */ u8 bStart; /* True if iDocid is valid */ u8 bDeferred; /* True if this expression is entirely deferred */ + + u32 *aMI; }; /* @@ -370,7 +381,6 @@ int sqlite3Fts3SegReaderNew(int, sqlite3_int64, int sqlite3Fts3SegReaderPending( Fts3Table*,int,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); -int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); @@ -382,7 +392,6 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); void sqlite3Fts3SegmentsClose(Fts3Table *); /* Special values interpreted by sqlite3SegReaderCursor() */ @@ -441,8 +450,7 @@ int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); -int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); +int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); @@ -480,9 +488,6 @@ int sqlite3Fts3TermSegReaderCursor( Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ); -int sqlite3Fts3EvalPhraseCache(Fts3Cursor *, Fts3Phrase *); -sqlite3_int64 sqlite3Fts3EvalDocid(Fts3Cursor *, Fts3Expr *); -int sqlite3Fts3EvalPhraseDoclist(Fts3Cursor*, Fts3Expr*, const char**,int*); void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index be40a9cfc2..0383d1a276 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -769,6 +769,7 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){ sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); sqlite3_free(p); } } diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 45e3c32f07..5ae3a16fc4 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -720,26 +720,6 @@ static int fts3ColumnlistCount(char **ppCollist){ return nEntry; } -static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){ - char *pCsr = *pp; - while( *pCsr ){ - int nHit; - sqlite3_int64 iCol = 0; - if( *pCsr==0x01 ){ - pCsr++; - pCsr += sqlite3Fts3GetVarint(pCsr, &iCol); - } - nHit = fts3ColumnlistCount(&pCsr); - assert( nHit>0 ); - if( isGlobal ){ - aOut[iCol*3+1]++; - } - aOut[iCol*3] += nHit; - } - pCsr++; - *pp = pCsr; -} - /* ** fts3ExprIterate() callback used to collect the "global" matchinfo stats ** for a single query. @@ -773,32 +753,9 @@ static int fts3ExprGlobalHitsCb( void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; - u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; - - if( pExpr->bDeferred ){ - int iCol; /* Column index */ - for(iCol=0; iColnCol; iCol++){ - aOut[iCol*3 + 1] = (u32)p->nDoc; - aOut[iCol*3 + 2] = (u32)p->nDoc; - } - }else{ - char *pIter; - char *pEnd; - int n; - int rc = sqlite3Fts3EvalPhraseDoclist( - p->pCursor, pExpr, (const char **)&pIter, &n - ); - if( rc!=SQLITE_OK ) return rc; - pEnd = &pIter[n]; - - /* Fill in the global hit count matrix row for this phrase. */ - while( pIterpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); } /* diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index a72556384c..30c0a1a15b 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -94,7 +94,6 @@ struct Fts3DeferredToken { ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() -** sqlite3Fts3SegReaderCost() ** sqlite3Fts3SegReaderIterate() ** ** Methods used to manipulate Fts3SegReader structures: @@ -1295,95 +1294,6 @@ static int fts3SegReaderNextDocid( return SQLITE_OK; } -/* -** This function is called to estimate the amount of data that will be -** loaded from the disk If SegReaderIterate() is called on this seg-reader, -** in units of average document size. -** -** This can be used as follows: If the caller has a small doclist that -** contains references to N documents, and is considering merging it with -** a large doclist (size X "average documents"), it may opt not to load -** the large doclist if X>N. -*/ -int sqlite3Fts3SegReaderCost( - Fts3Cursor *pCsr, /* FTS3 cursor handle */ - Fts3SegReader *pReader, /* Segment-reader handle */ - int *pnCost /* IN/OUT: Number of bytes read */ -){ - Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - int rc = SQLITE_OK; /* Return code */ - int nCost = 0; /* Cost in bytes to return */ - int pgsz = p->nPgsz; /* Database page size */ - - assert( pgsz>0 ); - - /* If this seg-reader is reading the pending-terms table, or if all data - ** for the segment is stored on the root page of the b-tree, then the cost - ** is zero. In this case all required data is already in main memory. - */ - if( p->bHasStat - && !fts3SegReaderIsPending(pReader) - && !fts3SegReaderIsRootOnly(pReader) - ){ - int nBlob = 0; - sqlite3_int64 iBlock; - - if( pCsr->nRowAvg==0 ){ - /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ - sqlite3_stmt *pStmt; - sqlite3_int64 nDoc = 0; - sqlite3_int64 nByte = 0; - const char *pEnd; - const char *a; - - rc = sqlite3Fts3SelectDoctotal(p, &pStmt); - if( rc!=SQLITE_OK ) return rc; - a = sqlite3_column_blob(pStmt, 0); - assert( a ); - - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz); - assert( pCsr->nRowAvg>0 ); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; - } - - /* Assume that a blob flows over onto overflow pages if it is larger - ** than (pgsz-35) bytes in size (the file-format documentation - ** confirms this). - */ - for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ - rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob, 0); - if( rc!=SQLITE_OK ) break; - if( (nBlob+35)>pgsz ){ - int nOvfl = (nBlob + 34)/pgsz; - nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); - } - } - } - - *pnCost += nCost; - return rc; -} int sqlite3Fts3MsrOvfl( Fts3Cursor *pCsr, @@ -2416,7 +2326,6 @@ int sqlite3Fts3MsrIncrNext( } while( 1 ){ - int nSort; Fts3SegReader *pSeg; pSeg = pMsr->apSegment[0]; @@ -2958,20 +2867,6 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ return rc; } -/* -** Return the deferred doclist associated with deferred token pDeferred. -** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already -** been called to allocate and populate the doclist. -*/ -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ - if( pDeferred->pList ){ - *pnByte = pDeferred->pList->nData; - return pDeferred->pList->aData; - } - *pnByte = 0; - return 0; -} - /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. diff --git a/manifest b/manifest index a97b87491f..c0c0a4d2ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sNEAR\squeries\suse\sincremental\smerging.\sFix\sissues\ssurrounding\sthe\sdeferred\stoken\soptimization. -D 2011-06-07T18:35:45.780 +C Fix\svarious\sissues\sto\sdo\swith\sdeferred\stokens,\sNEAR\sexpressions\sand\smatchinfo(). +D 2011-06-08T18:39:07.487 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 9d2d2cab4d64f0769046d88b6740c6e1f229d1e3 +F ext/fts3/fts3.c b44083cafb9840be0927f8b9fb2ab4f373167f77 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a +F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe -F ext/fts3/fts3_expr.c 0ae554230ada457e61e8184b24faac96aad78f6b +F ext/fts3/fts3_expr.c b95f0d76bcf4507c73a838f3178c4ed8c42dc2bb F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c 0485969cce410760b50d587a77186f9c7f7e96be +F ext/fts3/fts3_snippet.c 82e2c1e420c871c02f6e85ea438570118d7105c8 F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c ed525afd524d713abe7da174d56ad935dfc26008 +F ext/fts3/fts3_write.c bc24cec303d86aeb4b40fcbdf9f252f93ef78fc7 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -471,7 +471,7 @@ F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b -F test/fts3matchinfo.test f424597b6843659ecbc2009e8823380233ebf375 +F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 @@ -943,7 +943,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 567dd84359218245d4e6887547e2a48881f2c8e0 -R 020efe4a51ef4472e0e5c3f4175d0de6 +P 9d10a6846b12a9cc8fd4fdc3affd931a27218b5a +R 37e4da2cb9907d0ccf1d8076445165fd U dan -Z 740d1ddba83232619fca71041707ab60 +Z 147c4bbcabf01e6d99dff7a301984a70 diff --git a/manifest.uuid b/manifest.uuid index 4da09088b8..f3e1134a0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d10a6846b12a9cc8fd4fdc3affd931a27218b5a \ No newline at end of file +3972a787df5ec253b99b148385655e7b68d851fa \ No newline at end of file diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index e26bcf1052..40366b6aef 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -244,9 +244,13 @@ do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} } do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')"; -do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { - x {{5 8 2 5 5 5} {3 8 2 3 5 5}} - s {2 1} +# It used to be that the second 'a' token would be deferred. That doesn't +# work any longer. +if 0 { + do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { + x {{5 8 2 5 5 5} {3 8 2 3 5 5}} + s {2 1} + } } do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} } From 786b06896713494d7913976a590abe1c1420c30b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 9 Jun 2011 10:48:02 +0000 Subject: [PATCH 39/69] Fix problems to do with using both OR and NEAR operators in a single expression. FossilOrigin-Name: 4e8dd19eef04777d800977faf1859a405e396f30 --- ext/fts3/fts3.c | 71 ++++++++++++++++++++++++++++++----------------- manifest | 14 +++++----- manifest.uuid | 2 +- test/fts3rnd.test | 14 ++++++---- 4 files changed, 62 insertions(+), 39 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 5536a684ae..18a87e73ce 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3763,13 +3763,13 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ return rc; } -static void fts3EvalFreeDeferredDoclist(Fts3Phrase *pPhrase){ +static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ if( pPhrase->doclist.bFreeList ){ sqlite3_free(pPhrase->doclist.pList); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - pPhrase->doclist.bFreeList = 0; } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; } static int fts3EvalNearTrim2( @@ -3786,6 +3786,8 @@ static int fts3EvalNearTrim2( char *pOut; int res; + assert( pPhrase->doclist.pList ); + p2 = pOut = pPhrase->doclist.pList; res = fts3PoslistNearMerge( &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 @@ -3959,16 +3961,16 @@ static void fts3EvalNext( fts3EvalNext(pCsr, pRight, pRc); assert( *pRc!=SQLITE_OK || pRight->bStart ); } - do { - fts3EvalNext(pCsr, pLeft, pRc); - if( pLeft->bEof ) break; + + fts3EvalNext(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ while( !*pRc && !pRight->bEof && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 ){ fts3EvalNext(pCsr, pRight, pRc); } - }while( !pRight->bEof && pRight->iDocid==pLeft->iDocid && !*pRc ); + } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = pLeft->bEof; break; @@ -3976,7 +3978,7 @@ static void fts3EvalNext( default: { Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalFreeDeferredDoclist(pPhrase); + fts3EvalZeroPoslist(pPhrase); *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); pExpr->iDocid = pPhrase->doclist.iDocid; break; @@ -3997,11 +3999,11 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ && fts3EvalNearTest(pExpr, pRc) ); - /* If this is a NEAR node and the NEAR expression does not match - ** any rows, zero the doclist for all phrases involved in the NEAR. - ** This is because the snippet(), offsets() and matchinfo() functions - ** are not supposed to recognize any instances of phrases that are - ** part of unmatched NEAR queries. For example if this expression: + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: ** ** ... MATCH 'a OR (b NEAR c)' ** @@ -4012,12 +4014,19 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ ** then any snippet() should ony highlight the "a" term, not the "b" ** (as "b" is part of a non-matching NEAR clause). */ - if( pExpr->eType==FTSQUERY_NEAR && bHit==0 ){ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ Fts3Expr *p; for(p=pExpr; p->pPhrase==0; p=p->pLeft){ - p->pRight->pPhrase->doclist.pList = 0; + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pPhrase); } - p->pPhrase->doclist.pList = 0; } break; @@ -4037,9 +4046,14 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ break; default: { - if( pCsr->pDeferred ){ + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalFreeDeferredDoclist(pPhrase); + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalZeroPoslist(pPhrase); + } *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); bHit = (pPhrase->doclist.pList!=0); pExpr->iDocid = pCsr->iPrevId; @@ -4116,7 +4130,7 @@ static void fts3EvalRestart( Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ - fts3EvalFreeDeferredDoclist(pPhrase); + fts3EvalZeroPoslist(pPhrase); if( pPhrase->bIncr ){ sqlite3Fts3EvalPhraseCleanup(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); @@ -4172,7 +4186,7 @@ static void fts3EvalUpdateCounts( } } -static int fts3EvalNearStats( +static int fts3EvalGatherStats( Fts3Cursor *pCsr, Fts3Expr *pExpr ){ @@ -4195,6 +4209,7 @@ static int fts3EvalNearStats( } iDocid = pRoot->iDocid; bEof = pRoot->bEof; + assert( pRoot->bStart ); /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ for(p=pRoot; p; p=p->pLeft){ @@ -4236,11 +4251,17 @@ static int fts3EvalNearStats( if( bEof ){ pRoot->bEof = bEof; }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidiDocidbEof==0 ); - } + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); fts3EvalLoadDeferred(pCsr, &rc); } } @@ -4293,7 +4314,7 @@ int sqlite3Fts3EvalPhraseStats( aiOut[iCol*3 + 2] = pCsr->nDoc; } }else{ - rc = fts3EvalNearStats(pCsr, pExpr); + rc = fts3EvalGatherStats(pCsr, pExpr); if( rc==SQLITE_OK ){ assert( pExpr->aMI ); for(iCol=0; iColnColumn; iCol++){ @@ -4372,7 +4393,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ int i; sqlite3_free(pPhrase->doclist.aAll); - fts3EvalFreeDeferredDoclist(pPhrase); + fts3EvalZeroPoslist(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); for(i=0; inToken; i++){ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); diff --git a/manifest b/manifest index c0c0a4d2ec..4398869eb3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\svarious\sissues\sto\sdo\swith\sdeferred\stokens,\sNEAR\sexpressions\sand\smatchinfo(). -D 2011-06-08T18:39:07.487 +C Fix\sproblems\sto\sdo\swith\susing\sboth\sOR\sand\sNEAR\soperators\sin\sa\ssingle\sexpression. +D 2011-06-09T10:48:02.352 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c b44083cafb9840be0927f8b9fb2ab4f373167f77 +F ext/fts3/fts3.c 5df3b5797522d3d17949ee12d5918d6d213b5114 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -475,7 +475,7 @@ F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 -F test/fts3rnd.test d88ec3dbe52e81e410cd1a39554d15941f86333c +F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 @@ -943,7 +943,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 9d10a6846b12a9cc8fd4fdc3affd931a27218b5a -R 37e4da2cb9907d0ccf1d8076445165fd +P 3972a787df5ec253b99b148385655e7b68d851fa +R 9f88ec7fe13b9820fcb74aa4a46dd50c U dan -Z 147c4bbcabf01e6d99dff7a301984a70 +Z 78f71a9a33e1daee77ff848d72b67bf4 diff --git a/manifest.uuid b/manifest.uuid index f3e1134a0a..f91e3008ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3972a787df5ec253b99b148385655e7b68d851fa \ No newline at end of file +4e8dd19eef04777d800977faf1859a405e396f30 \ No newline at end of file diff --git a/test/fts3rnd.test b/test/fts3rnd.test index 0ad41d8ce4..97af54925f 100644 --- a/test/fts3rnd.test +++ b/test/fts3rnd.test @@ -273,6 +273,8 @@ proc do_orderbydocid_test {tn sql res} { ] } +set NUM_TRIALS 100 + foreach {nodesize order} { 50 DESC 50 ASC @@ -292,7 +294,7 @@ foreach {nodesize order} { for {set i 0} {$i < 100} {incr i} { insert_row $i } } - for {set iTest 1} {$iTest <= 100} {incr iTest} { + for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} { catchsql COMMIT set DO_MALLOC_TEST 0 @@ -334,11 +336,11 @@ foreach {nodesize order} { # for {set i 0} {$i < 10} {incr i} { set term [random_term] - do_select_test 1.$i { + do_select_test 1.$i.asc { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term ORDER BY docid ASC } [simple_token_matchinfo $term 0] - do_select_test 1.$i { + do_select_test 1.$i.desc { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term ORDER BY docid DESC } [simple_token_matchinfo $term 1] @@ -432,9 +434,9 @@ foreach {nodesize order} { # Set operations on NEAR queries. # foreach {tn op proc} { - 8 OR setop_or - 9 NOT setop_not - 10 AND setop_and + 11 OR setop_or + 12 NOT setop_not + 13 AND setop_and } { for {set i 0} {$i < $nRep} {incr i} { set term1 [random_term] From 9dde7cbbadbfc2f60122bb3a4e06c7a89414c9c3 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 9 Jun 2011 17:53:43 +0000 Subject: [PATCH 40/69] Fix a line in pcache1.c where a global data structure is accessed without using the GLOBAL() macro. This causes a subtle malfunction on test systems that use SQLITE_OMIT_WSD. FossilOrigin-Name: b11b2e1f8ccadf78bebe2278f05a8e3d3e543328 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pcache1.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c598385e18..ded9d9cb63 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scomment\stype\son\sthe\sdescription\sof\sthe\sSchema\sobject. -D 2011-06-07T18:31:14.409 +C Fix\sa\sline\sin\spcache1.c\swhere\sa\sglobal\sdata\sstructure\sis\saccessed\swithout\susing\sthe\sGLOBAL()\smacro.\sThis\scauses\sa\ssubtle\smalfunction\son\stest\ssystems\sthat\suse\sSQLITE_OMIT_WSD. +D 2011-06-09T17:53:43.990 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -170,7 +170,7 @@ F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 -F src/pcache1.c d548e31beafa792d1994b663a29a5303569efc4e +F src/pcache1.c 912bd5687d6df344698d8e69560f347b6e21c18a F src/pragma.c ebcd20f1e654f5cb3aeef864ed69c4697719fbaa F src/prepare.c e64261559a3187698a3e7e6c8b001a4f4f98dab4 F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 98ccfa930e8e5f10808a518e3e22e85e8a8a65c2 -R eade3cd28243c0adfa4f9b0e3371ab06 -U drh -Z 6c3c2f88a9e5ad93debb108bda4e78cb +P 095cd9a6ec175b703ff3fcafeffb3349f21bd831 +R 7c10b21d6dc959e0ea4f54033f3d408b +U dan +Z e98f319e77bc2f43df3a70aff2c9cf53 diff --git a/manifest.uuid b/manifest.uuid index df6972f87d..4449540538 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -095cd9a6ec175b703ff3fcafeffb3349f21bd831 \ No newline at end of file +b11b2e1f8ccadf78bebe2278f05a8e3d3e543328 \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index ad443954bf..e47265a225 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -574,7 +574,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){ pGroup = (PGroup*)&pCache[1]; pGroup->mxPinned = 10; }else{ - pGroup = &pcache1_g.grp; + pGroup = &pcache1.grp; } pCache->pGroup = pGroup; pCache->szPage = szPage; From b328debc6a035a9c91306b5a3bf95aea2fc564be Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Jun 2011 16:33:25 +0000 Subject: [PATCH 41/69] Fix minor problems with foreign key constraints where the parent table is the same as the child table. FossilOrigin-Name: 442d8d8bfe443797482354ba8766d97d3d6acaae --- manifest | 14 ++++----- manifest.uuid | 2 +- src/fkey.c | 14 ++++++++- test/fkey3.test | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ded9d9cb63..79636217b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sline\sin\spcache1.c\swhere\sa\sglobal\sdata\sstructure\sis\saccessed\swithout\susing\sthe\sGLOBAL()\smacro.\sThis\scauses\sa\ssubtle\smalfunction\son\stest\ssystems\sthat\suse\sSQLITE_OMIT_WSD. -D 2011-06-09T17:53:43.990 +C Fix\sminor\sproblems\swith\sforeign\skey\sconstraints\swhere\sthe\sparent\stable\sis\sthe\ssame\sas\sthe\schild\stable. +D 2011-06-10T16:33:25.121 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,7 +133,7 @@ F src/date.c 1548fdac51377e4e7833251de878b4058c148e1b F src/delete.c cecc926c70783452f3e8eb452c728291ce1a0b21 F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c a43ba8a005fb5efd1deeee06853e3a6120d46a91 +F src/fkey.c 9fabba17a4d4778dc660f0cb9d781fc86d7b9d41 F src/func.c b9117e40975245b8504cf3625d7e321d8d4b63dc F src/global.c 29bfb85611dd816b04f10fba0ca910366e128d38 F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af @@ -398,7 +398,7 @@ F test/filectrl.test 97003734290887566e01dded09dc9e99cb937e9e F test/filefmt.test f178cfc29501a14565954c961b226e61877dd32c F test/fkey1.test 01c7de578e11747e720c2d9aeef27f239853c4da F test/fkey2.test 080969fe219b3b082b0e097ac18c6af2e5b0631f -F test/fkey3.test 42f88d6048d8dc079e2a8cf7baad1cc1483a7620 +F test/fkey3.test 0c4d36b6d5b88f2c233cf8a512d3e2eaedc06fd6 F test/fkey4.test c6c8f9f9be885f95c85c7bceb26f243ad906fd49 F test/fkey_malloc.test a5ede29bd2f6e56dea78c3d43fb86dd696c068c8 F test/format4.test 1f0cac8ff3895e9359ed87e41aaabee982a812eb @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 095cd9a6ec175b703ff3fcafeffb3349f21bd831 -R 7c10b21d6dc959e0ea4f54033f3d408b +P b11b2e1f8ccadf78bebe2278f05a8e3d3e543328 +R 82679520a5c6b1fbd50bacf672964c79 U dan -Z e98f319e77bc2f43df3a70aff2c9cf53 +Z 3e44af2707cf5eb06c1e283bd2e9d2ca diff --git a/manifest.uuid b/manifest.uuid index 4449540538..3b5ddc572a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b11b2e1f8ccadf78bebe2278f05a8e3d3e543328 \ No newline at end of file +442d8d8bfe443797482354ba8766d97d3d6acaae \ No newline at end of file diff --git a/src/fkey.c b/src/fkey.c index 34fdfda5d9..dda8a50d44 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -386,13 +386,25 @@ static void fkLookupParent( /* If the parent table is the same as the child table, and we are about ** to increment the constraint-counter (i.e. this is an INSERT operation), ** then check if the row being inserted matches itself. If so, do not - ** increment the constraint-counter. */ + ** increment the constraint-counter. + ** + ** If any of the parent-key values are NULL, then the row cannot match + ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any + ** of the parent-key values are NULL (at this point it is known that + ** none of the child key values are). + */ if( pTab==pFKey->pFrom && nIncr==1 ){ int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; for(i=0; iaiColumn[i]+1+regData; + assert( aiCol[i]!=pTab->iPKey ); + if( pIdx->aiColumn[i]==pTab->iPKey ){ + /* The parent key is a composite key that includes the IPK column */ + iParent = regData; + } sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); } sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); } diff --git a/test/fkey3.test b/test/fkey3.test index 88b5aad370..2df3be530e 100644 --- a/test/fkey3.test +++ b/test/fkey3.test @@ -21,6 +21,8 @@ ifcapable {!foreignkey||!trigger} { return } +set testprefix fkey3 + # Create a table and some data to work with. # do_test fkey3-1.1 { @@ -77,4 +79,79 @@ do_test fkey3-2.1 { } } {1 100 1 101 2 100 2 101} + +#------------------------------------------------------------------------- +# The following tests - fkey-3.* - test some edge cases to do with +# inserting rows into tables that have foreign keys where the parent +# table is the same as the child table. Especially cases where the +# new row being inserted matches itself. +# +do_execsql_test 3.1.1 { + CREATE TABLE t3(a, b, c, d, + UNIQUE(a, b), + FOREIGN KEY(c, d) REFERENCES t3(a, b) + ); + INSERT INTO t3 VALUES(1, 2, 1, 2); +} {} +do_catchsql_test 3.1.2 { + INSERT INTO t3 VALUES(NULL, 2, 5, 2); +} {1 {foreign key constraint failed}} +do_catchsql_test 3.1.3 { + INSERT INTO t3 VALUES(NULL, 3, 5, 2); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.2.1 { + CREATE TABLE t4(a UNIQUE, b REFERENCES t4(a)); +} +do_catchsql_test 3.2.2 { + INSERT INTO t4 VALUES(NULL, 1); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.3.1 { + CREATE TABLE t5(a INTEGER PRIMARY KEY, b REFERENCES t5(a)); + INSERT INTO t5 VALUES(NULL, 1); +} {} +do_catchsql_test 3.3.2 { + INSERT INTO t5 VALUES(NULL, 3); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.4.1 { + CREATE TABLE t6(a INTEGER PRIMARY KEY, b, c, d, + FOREIGN KEY(c, d) REFERENCES t6(a, b) + ); + CREATE UNIQUE INDEX t6i ON t6(b, a); +} +do_execsql_test 3.4.2 { INSERT INTO t6 VALUES(NULL, 'a', 1, 'a'); } {} +do_execsql_test 3.4.3 { INSERT INTO t6 VALUES(2, 'a', 2, 'a'); } {} +do_execsql_test 3.4.4 { INSERT INTO t6 VALUES(NULL, 'a', 1, 'a'); } {} +do_execsql_test 3.4.5 { INSERT INTO t6 VALUES(5, 'a', 2, 'a'); } {} +do_catchsql_test 3.4.6 { + INSERT INTO t6 VALUES(NULL, 'a', 65, 'a'); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.4.7 { + INSERT INTO t6 VALUES(100, 'one', 100, 'one'); + DELETE FROM t6 WHERE a = 100; +} +do_execsql_test 3.4.8 { + INSERT INTO t6 VALUES(100, 'one', 100, 'one'); + UPDATE t6 SET c = 1, d = 'a' WHERE a = 100; + DELETE FROM t6 WHERE a = 100; +} + +do_execsql_test 3.5.1 { + CREATE TABLE t7(a, b, c, d INTEGER PRIMARY KEY, + FOREIGN KEY(c, d) REFERENCES t7(a, b) + ); + CREATE UNIQUE INDEX t7i ON t7(a, b); +} +do_execsql_test 3.5.2 { INSERT INTO t7 VALUES('x', 1, 'x', NULL) } {} +do_execsql_test 3.5.3 { INSERT INTO t7 VALUES('x', 2, 'x', 2) } {} +do_catchsql_test 3.5.4 { + INSERT INTO t7 VALUES('x', 450, 'x', NULL); +} {1 {foreign key constraint failed}} +do_catchsql_test 3.5.5 { + INSERT INTO t7 VALUES('x', 450, 'x', 451); +} {1 {foreign key constraint failed}} + finish_test From 7006c18e0043a2787b763e7fa7f3a0df964fe5cf Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Jun 2011 18:33:35 +0000 Subject: [PATCH 42/69] When updating a field that requires foreign key constraints be checked, ensure that the indexes and tables are consistent when the FK logic is run. Otherwise, it may detect the inconsistency and report database corruption. FossilOrigin-Name: 2b3d9996a829c62fbaf7c92d50e44636340b07c6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/update.c | 2 +- test/fkey3.test | 29 +++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 79636217b0..01e7bf60e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\sproblems\swith\sforeign\skey\sconstraints\swhere\sthe\sparent\stable\sis\sthe\ssame\sas\sthe\schild\stable. -D 2011-06-10T16:33:25.121 +C When\supdating\sa\sfield\sthat\srequires\sforeign\skey\sconstraints\sbe\schecked,\sensure\sthat\sthe\sindexes\sand\stables\sare\sconsistent\swhen\sthe\sFK\slogic\sis\srun.\sOtherwise,\sit\smay\sdetect\sthe\sinconsistency\sand\sreport\sdatabase\scorruption. +D 2011-06-10T18:33:35.602 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -232,7 +232,7 @@ F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 F src/trigger.c c836a6caac16ba96611558922106858f6ca3d6bf -F src/update.c 5bcb56e5c7380a2eecb0e71891dbd4ad7437748f +F src/update.c 80d77311d91ebc06b27149e75701f1b3e9356622 F src/utf.c d83650c3ea08f7407bd9d0839d9885241c209c60 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e @@ -398,7 +398,7 @@ F test/filectrl.test 97003734290887566e01dded09dc9e99cb937e9e F test/filefmt.test f178cfc29501a14565954c961b226e61877dd32c F test/fkey1.test 01c7de578e11747e720c2d9aeef27f239853c4da F test/fkey2.test 080969fe219b3b082b0e097ac18c6af2e5b0631f -F test/fkey3.test 0c4d36b6d5b88f2c233cf8a512d3e2eaedc06fd6 +F test/fkey3.test 5ec899d12b13bcf1e9ef40eff7fb692fdb91392e F test/fkey4.test c6c8f9f9be885f95c85c7bceb26f243ad906fd49 F test/fkey_malloc.test a5ede29bd2f6e56dea78c3d43fb86dd696c068c8 F test/format4.test 1f0cac8ff3895e9359ed87e41aaabee982a812eb @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P b11b2e1f8ccadf78bebe2278f05a8e3d3e543328 -R 82679520a5c6b1fbd50bacf672964c79 +P 442d8d8bfe443797482354ba8766d97d3d6acaae +R dc98bce2b79b015d80463a3aa5591d5c U dan -Z 3e44af2707cf5eb06c1e283bd2e9d2ca +Z 8380c49075a7949acdc7a8820572b0c9 diff --git a/manifest.uuid b/manifest.uuid index 3b5ddc572a..c85de3778b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -442d8d8bfe443797482354ba8766d97d3d6acaae \ No newline at end of file +2b3d9996a829c62fbaf7c92d50e44636340b07c6 \ No newline at end of file diff --git a/src/update.c b/src/update.c index aecf75cf35..d445f01f4e 100644 --- a/src/update.c +++ b/src/update.c @@ -244,7 +244,7 @@ void sqlite3Update( } for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; - if( chngRowid ){ + if( hasFK || chngRowid ){ reg = ++pParse->nMem; }else{ reg = 0; diff --git a/test/fkey3.test b/test/fkey3.test index 2df3be530e..6a11f8883e 100644 --- a/test/fkey3.test +++ b/test/fkey3.test @@ -154,4 +154,33 @@ do_catchsql_test 3.5.5 { INSERT INTO t7 VALUES('x', 450, 'x', 451); } {1 {foreign key constraint failed}} + +do_execsql_test 3.6.1 { + CREATE TABLE t8(a, b, c, d, e, FOREIGN KEY(c, d) REFERENCES t8(a, b)); + CREATE UNIQUE INDEX t8i1 ON t8(a, b); + CREATE UNIQUE INDEX t8i2 ON t8(c); + INSERT INTO t8 VALUES(1, 1, 1, 1, 1); +} +do_catchsql_test 3.6.2 { + UPDATE t8 SET d = 2; +} {1 {foreign key constraint failed}} +do_execsql_test 3.6.3 { UPDATE t8 SET d = 1; } +do_execsql_test 3.6.4 { UPDATE t8 SET e = 2; } + +do_catchsql_test 3.6.5 { + CREATE TABLE TestTable ( + id INTEGER PRIMARY KEY, + name text, + source_id integer not null, + parent_id integer, + + foreign key(source_id, parent_id) references TestTable(source_id, id) + ); + CREATE UNIQUE INDEX testindex on TestTable(source_id, id); + PRAGMA foreign_keys=1; + INSERT INTO TestTable VALUES (1, 'parent', 1, null); + INSERT INTO TestTable VALUES (2, 'child', 1, 1); + UPDATE TestTable SET parent_id=1000 where id=2; +} {1 {foreign key constraint failed}} + finish_test From 99ebad90e3582641eb388daf9e440cb020770c8d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Jun 2011 09:11:01 +0000 Subject: [PATCH 43/69] Fix a bug exposed by combining matchinfo(), NEAR and "ORDER BY rowid DESC". FossilOrigin-Name: 5f6b87f420f21749aa7c72e020c50aca74890086 --- Makefile.in | 3 +- ext/fts3/fts3.c | 7 +- ext/fts3/fts3_test.c | 249 +++++++++++++++++++++++++++++++++++++++++++ main.mk | 1 + manifest | 22 ++-- manifest.uuid | 2 +- src/tclsqlite.c | 8 ++ test/fts3auto.test | 201 ++++++++++++++++++++++++++++++++++ test/tester.tcl | 4 +- 9 files changed, 481 insertions(+), 16 deletions(-) create mode 100644 ext/fts3/fts3_test.c create mode 100644 test/fts3auto.test diff --git a/Makefile.in b/Makefile.in index b0871ad940..4a81e15664 100644 --- a/Makefile.in +++ b/Makefile.in @@ -379,7 +379,8 @@ TESTSRC = \ $(TOP)/src/test_vfs.c \ $(TOP)/src/test_wholenumber.c \ $(TOP)/src/test_wsd.c \ - $(TOP)/ext/fts3/fts3_term.c + $(TOP)/ext/fts3/fts3_term.c \ + $(TOP)/ext/fts3/fts3_test.c # Source code to the library files needed by the test fixture # diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 18a87e73ce..4b37c4e645 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2727,8 +2727,10 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ ** same position list. */ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ - char *p = &(*ppPoslist)[-3]; - char c = p[1]; + char *p = &(*ppPoslist)[-2]; + char c; + + while( p>pStart && (c=*p--)==0 ); while( p>pStart && (*p & 0x80) | c ){ c = *p--; } @@ -3422,6 +3424,7 @@ void sqlite3Fts3DoclistPrev( iDocid += (iMul * iDelta); pNext = pDocid; fts3PoslistCopy(0, &pDocid); + while( pDocid +#include +#include + +#define NM_MAX_TOKEN 12 + +typedef struct NearPhrase NearPhrase; +typedef struct NearDocument NearDocument; +typedef struct NearToken NearToken; + +struct NearDocument { + int nToken; /* Length of token in bytes */ + NearToken *aToken; /* Token array */ +}; + +struct NearToken { + int n; /* Length of token in bytes */ + const char *z; /* Pointer to token string */ +}; + +struct NearPhrase { + int nNear; /* Preceding NEAR value */ + int nToken; /* Number of tokens in this phrase */ + NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */ +}; + +static int nm_phrase_match( + NearPhrase *p, + NearToken *aToken +){ + int ii; + + for(ii=0; iinToken; ii++){ + NearToken *pToken = &p->aToken[ii]; + if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){ + if( aToken[ii].n<(pToken->n-1) ) return 0; + if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0; + }else{ + if( aToken[ii].n!=pToken->n ) return 0; + if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0; + } + } + + return 1; +} + +static int nm_near_chain( + int iDir, /* Direction to iterate through aPhrase[] */ + NearDocument *pDoc, /* Document to match against */ + int iPos, /* Position at which iPhrase was found */ + int nPhrase, /* Size of phrase array */ + NearPhrase *aPhrase, /* Phrase array */ + int iPhrase /* Index of phrase found */ +){ + int iStart; + int iStop; + int ii; + int nNear; + int iPhrase2; + NearPhrase *p; + NearPhrase *pPrev; + + assert( iDir==1 || iDir==-1 ); + + if( iDir==1 ){ + if( (iPhrase+1)==nPhrase ) return 1; + nNear = aPhrase[iPhrase+1].nNear; + }else{ + if( iPhrase==0 ) return 1; + nNear = aPhrase[iPhrase].nNear; + } + pPrev = &aPhrase[iPhrase]; + iPhrase2 = iPhrase+iDir; + p = &aPhrase[iPhrase2]; + + iStart = iPos - nNear - p->nToken; + iStop = iPos + nNear + pPrev->nToken; + + if( iStart<0 ) iStart = 0; + if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken; + + for(ii=iStart; ii<=iStop; ii++){ + if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ + if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1; + } + } + + return 0; +} + +static int nm_match_count( + NearDocument *pDoc, /* Document to match against */ + int nPhrase, /* Size of phrase array */ + NearPhrase *aPhrase, /* Phrase array */ + int iPhrase /* Index of phrase to count matches for */ +){ + int nOcc = 0; + int ii; + NearPhrase *p = &aPhrase[iPhrase]; + + for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){ + if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ + /* Test forward NEAR chain (i>iPhrase) */ + if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue; + + /* Test reverse NEAR chain (iNM_MAX_TOKEN ){ + Tcl_AppendResult(interp, "Too many tokens in phrase", 0); + rc = TCL_ERROR; + goto near_match_out; + } + for(jj=0; jjz = Tcl_GetStringFromObj(apToken[jj], &pT->n); + } + aPhrase[ii].nToken = nToken; + } + for(ii=1; ii0)); + + near_match_out: + ckfree((char *)aPhrase); + ckfree((char *)doc.aToken); + return rc; +} + +int Sqlitetestfts3_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); + return TCL_OK; +} + diff --git a/main.mk b/main.mk index f84e0dea8b..ec743be439 100644 --- a/main.mk +++ b/main.mk @@ -220,6 +220,7 @@ SRC += \ # Source code to the test files. # TESTSRC = \ + $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/src/test1.c \ $(TOP)/src/test2.c \ $(TOP)/src/test3.c \ diff --git a/manifest b/manifest index 4398869eb3..c96551fae9 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Fix\sproblems\sto\sdo\swith\susing\sboth\sOR\sand\sNEAR\soperators\sin\sa\ssingle\sexpression. -D 2011-06-09T10:48:02.352 +C Fix\sa\sbug\sexposed\sby\scombining\smatchinfo(),\sNEAR\sand\s"ORDER\sBY\srowid\sDESC". +D 2011-06-13T09:11:01.953 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d +F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 5df3b5797522d3d17949ee12d5918d6d213b5114 +F ext/fts3/fts3.c e71dafb1f324358d12fd02ea12644d8c6cea577a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -72,6 +72,7 @@ F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 F ext/fts3/fts3_snippet.c 82e2c1e420c871c02f6e85ea438570118d7105c8 F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 +F ext/fts3/fts3_test.c 9376cc865447e63c671f0f9ffd1a2c9a29678230 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d @@ -102,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 6111163d4e9720e4212ef288e967b4aa2c2ce379 +F main.mk 6de0d92dcae3d399a6bafaeb23b57f6ef0f41955 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -185,7 +186,7 @@ F src/sqliteInt.h 6e58c558c57c8f44011736d5fa5295eb3130f9de F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 501c9a200fd998a268be475be5858febc90b725b +F src/tclsqlite.c c83f5b4a15ed92cb35aa04320aa4a30b95071b2f F src/test1.c efca486a25fb894988e7a82e84579a4e57388a02 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 @@ -454,6 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 +F test/fts3auto.test 696a2d32dd64a03aa47818c26ea64f8f27e7eb07 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -692,7 +694,7 @@ F test/tclsqlite.test 8c154101e704170c2be10f137a5499ac2c6da8d3 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 -F test/tester.tcl a791ee74cb6b8f8613079ccc018bf2c8b952a26c +F test/tester.tcl 76222602e59047c6ef119473c7a2ea7c6ee73d09 F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f F test/thread002.test afd20095e6e845b405df4f2c920cb93301ca69db F test/thread003.test b824d4f52b870ae39fc5bae4d8070eca73085dca @@ -943,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 3972a787df5ec253b99b148385655e7b68d851fa -R 9f88ec7fe13b9820fcb74aa4a46dd50c +P 4e8dd19eef04777d800977faf1859a405e396f30 +R 6b4754b974de210b10ad796c5876a1bc U dan -Z 78f71a9a33e1daee77ff848d72b67bf4 +Z f1c93614cce7c70636f14d06ee6a3496 diff --git a/manifest.uuid b/manifest.uuid index f91e3008ee..5c8deb0eac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e8dd19eef04777d800977faf1859a405e396f30 \ No newline at end of file +5f6b87f420f21749aa7c72e020c50aca74890086 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 575651d7e5..49ff8bcd8d 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3585,6 +3585,10 @@ static void init_all(Tcl_Interp *interp){ extern int Sqlitetestfuzzer_Init(Tcl_Interp*); extern int Sqlitetestwholenumber_Init(Tcl_Interp*); +#ifdef SQLITE_ENABLE_FTS3 + extern int Sqlitetestfts3_Init(Tcl_Interp *interp); +#endif + #ifdef SQLITE_ENABLE_ZIPVFS extern int Zipvfs_Init(Tcl_Interp*); Zipvfs_Init(interp); @@ -3625,6 +3629,10 @@ static void init_all(Tcl_Interp *interp){ Sqlitetestfuzzer_Init(interp); Sqlitetestwholenumber_Init(interp); +#ifdef SQLITE_ENABLE_FTS3 + Sqlitetestfts3_Init(interp); +#endif + Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0); #ifdef SQLITE_SSE diff --git a/test/fts3auto.test b/test/fts3auto.test new file mode 100644 index 0000000000..fe87d9f2c7 --- /dev/null +++ b/test/fts3auto.test @@ -0,0 +1,201 @@ +# 2011 June 10 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl +source $testdir/malloc_common.tcl + +set testprefix fts3rnd2 + +proc test_fts3_near_match {tn doc expr res} { + fts3_near_match $doc $expr -phrasecountvar p + uplevel do_test [list $tn] [list [list set {} $p]] [list $res] +} + +# Simple test cases for C routine [fts3_near_match]. +# +test_fts3_near_match 1.1.1 {a b c a b} a {2} +test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c} {2 2 1} +test_fts3_near_match 1.1.3 {a b c a b} {"a b"} {2} +test_fts3_near_match 1.1.4 {a b c a b} {"b c"} {1} +test_fts3_near_match 1.1.5 {a b c a b} {"c c"} {0} + +test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f} {0 0} +test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f} {1 1} +test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b} {0 0} +test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b} {1 1} +test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0} +test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1} + +set A "a b c d e f g h i j k l m n o p q r s t u v w x y z" +test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0} +test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + +proc fix_near_expr {expr} { + set out [list] + lappend out [lindex $expr 0] + foreach {a b} [lrange $expr 1 end] { + if {[string match -nocase near $a]} { set a 10 } + if {[string match -nocase near/* $a]} { set a [string range $a 5 end] } + lappend out $a + lappend out $b + } + return $out +} + +proc do_near_test {tn tbl expr} { + + set expr [fix_near_expr $expr] + + # Create the MATCH expression from $expr + # + set match [lindex $expr 0] + if {[llength $match]>1} { + set match "\"$match\"" + } + foreach {nNear phrase} [lrange $expr 1 end] { + if {[llength $phrase]>1} { + append match " NEAR/$nNear \"$phrase\"" + } else { + append match " NEAR/$nNear $phrase" + } + } + + # Calculate the expected results using [fts3_near_match]. The following + # loop populates the "hits" and "counts" arrays as follows: + # + # 1. For each document in the table that matches the NEAR expression, + # hits($docid) is set to 1. The set of docids that match the expression + # can therefore be found using [array names hits]. + # + # 2. For each column of each document in the table, counts($docid,$iCol) + # is set to the -phrasecountvar output. + # + set res [list] + catch { array unset hits } + db eval "SELECT docid, * FROM $tbl" d { + set iCol 0 + foreach col [lrange $d(*) 1 end] { + set docid $d(docid) + set hit [fts3_near_match $d($col) $expr -p counts($docid,$iCol)] + if {$hit} { set hits($docid) 1 } + incr iCol + } + } + set nPhrase [expr ([llength $expr]+1)/2] + set nCol $iCol + + # This block populates the nHit and nDoc arrays. For each phrase/column + # in the query/table, array elements are set as follows: + # + # nHit($iPhrase,$iCol) - Total number of hits for phrase $iPhrase in + # column $iCol. + # + # nDoc($iPhrase,$iCol) - Number of documents with at least one hit for + # phrase $iPhrase in column $iCol. + # + for {set iPhrase 0} {$iPhrase < $nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol < $nCol} {incr iCol} { + set nHit($iPhrase,$iCol) 0 + set nDoc($iPhrase,$iCol) 0 + } + } + foreach key [array names counts] { + set iCol [lindex [split $key ,] 1] + set iPhrase 0 + foreach c $counts($key) { + if {$c>0} { incr nHit($iPhrase,$iCol) 1 } + incr nDoc($iPhrase,$iCol) $c + incr iPhrase + } + } + + # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to + # contain the output of matchinfo('x') for the document. + # + foreach docid [array names hits] { + set mi [list] + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol<$nCol} {incr iCol} { + lappend mi [lindex $counts($docid,$iCol) $iPhrase] + lappend mi $nDoc($iPhrase,$iCol) + lappend mi $nHit($iPhrase,$iCol) + } + } + set aMatchinfo($docid) $mi + } + + set matchinfo_asc [list] + foreach docid [lsort -integer -incr [array names aMatchinfo]] { + lappend matchinfo_asc $docid $aMatchinfo($docid) + } + set matchinfo_desc [list] + foreach docid [lsort -integer -decr [array names aMatchinfo]] { + lappend matchinfo_desc $docid $aMatchinfo($docid) + } + + set title "(\"$match\" -> [llength [array names hits]] rows)" + + do_execsql_test $tn$title.1 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC + " [lsort -integer -incr [array names hits]] + + do_execsql_test $tn$title.2 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC + " [lsort -integer -decr [array names hits]] + + do_execsql_test $tn$title.3 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid DESC + " $matchinfo_desc + + do_execsql_test $tn$title.4 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid ASC + " $matchinfo_asc +} + +do_test 2.1 { + execsql { CREATE VIRTUAL TABLE t1 USING fts3(a, b) } + for {set i 0} {$i<32} {incr i} { + set doc [list] + if {$i&0x01} {lappend doc one} + if {$i&0x02} {lappend doc two} + if {$i&0x04} {lappend doc three} + if {$i&0x08} {lappend doc four} + if {$i&0x10} {lappend doc five} + execsql { INSERT INTO t1 VALUES($doc, null) } + } +} {} +foreach {tn expr} { + 1 {one} + 2 {one NEAR/1 five} + 3 {t*} + 4 {t* NEAR/0 five} + 5 {o* NEAR/1 f*} + 6 {one NEAR five NEAR two NEAR four NEAR three} +} { + do_near_test 2.2.$tn t1 $expr +} + +finish_test + diff --git a/test/tester.tcl b/test/tester.tcl index 0b66c3d788..a53723b9a1 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -374,11 +374,11 @@ proc fix_testname {varname} { proc do_execsql_test {testname sql {result {}}} { fix_testname testname - uplevel do_test $testname [list "execsql {$sql}"] [list [list {*}$result]] + uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]] } proc do_catchsql_test {testname sql result} { fix_testname testname - uplevel do_test $testname [list "catchsql {$sql}"] [list $result] + uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result] } proc do_eqp_test {name sql res} { uplevel do_execsql_test $name [list "EXPLAIN QUERY PLAN $sql"] [list $res] From 0a32fa6d8150d8b5399a87d6c3ee7a679ee241a6 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Jun 2011 12:19:21 +0000 Subject: [PATCH 44/69] Use only unsigned values in the implementatin of LIKE and GLOB so that values won't overflow to negative when dealing with malformed UTF8. FossilOrigin-Name: 77f01578bb565d1bc884b374b68bae10ce34a084 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/func.c | 12 ++++++------ src/sqliteInt.h | 2 +- src/utf.c | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 01e7bf60e7..404f96248e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\supdating\sa\sfield\sthat\srequires\sforeign\skey\sconstraints\sbe\schecked,\sensure\sthat\sthe\sindexes\sand\stables\sare\sconsistent\swhen\sthe\sFK\slogic\sis\srun.\sOtherwise,\sit\smay\sdetect\sthe\sinconsistency\sand\sreport\sdatabase\scorruption. -D 2011-06-10T18:33:35.602 +C Use\sonly\sunsigned\svalues\sin\sthe\simplementatin\sof\sLIKE\sand\sGLOB\sso\sthat\nvalues\swon't\soverflow\sto\snegative\swhen\sdealing\swith\smalformed\sUTF8. +D 2011-06-13T12:19:21.072 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -134,7 +134,7 @@ F src/delete.c cecc926c70783452f3e8eb452c728291ce1a0b21 F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 9fabba17a4d4778dc660f0cb9d781fc86d7b9d41 -F src/func.c b9117e40975245b8504cf3625d7e321d8d4b63dc +F src/func.c d93772d9ffa51e4a8f275675bae1f61394c4ab80 F src/global.c 29bfb85611dd816b04f10fba0ca910366e128d38 F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 @@ -181,7 +181,7 @@ F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 F src/sqlite.h.in 2f51e4f58b2b4626fcbd9938580e730cb5fb4985 F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h d2a9f6e06b85bb72a47cfe8d45320abe9cfa44f1 +F src/sqliteInt.h 7b7ec2394b94fc4516930cd9dae37af0f9312215 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -233,7 +233,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 F src/trigger.c c836a6caac16ba96611558922106858f6ca3d6bf F src/update.c 80d77311d91ebc06b27149e75701f1b3e9356622 -F src/utf.c d83650c3ea08f7407bd9d0839d9885241c209c60 +F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e F src/vdbe.c edfa3827d7a6fac2425bc10c0eb6e54342d2fa56 @@ -942,7 +942,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 442d8d8bfe443797482354ba8766d97d3d6acaae -R dc98bce2b79b015d80463a3aa5591d5c -U dan -Z 8380c49075a7949acdc7a8820572b0c9 +P 2b3d9996a829c62fbaf7c92d50e44636340b07c6 +R 901f548a763eaba41771c9603ccfd6ef +U drh +Z 700a00fc01c336d2e1728d856de33900 diff --git a/manifest.uuid b/manifest.uuid index c85de3778b..37f82214ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b3d9996a829c62fbaf7c92d50e44636340b07c6 \ No newline at end of file +77f01578bb565d1bc884b374b68bae10ce34a084 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 0b9b600d79..407ea26aa6 100644 --- a/src/func.c +++ b/src/func.c @@ -506,10 +506,10 @@ struct compareInfo { ** whereas only characters less than 0x80 do in ASCII. */ #if defined(SQLITE_EBCDIC) -# define sqlite3Utf8Read(A,C) (*(A++)) -# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] +# define sqlite3Utf8Read(A,C) (*(A++)) +# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] #else -# define GlogUpperToLower(A) if( A<0x80 ){ A = sqlite3UpperToLower[A]; } +# define GlogUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -552,9 +552,9 @@ static int patternCompare( const u8 *zPattern, /* The glob pattern */ const u8 *zString, /* The string to compare against the glob */ const struct compareInfo *pInfo, /* Information about how to do the compare */ - const int esc /* The escape character */ + u32 esc /* The escape character */ ){ - int c, c2; + u32 c, c2; int invert; int seen; u8 matchOne = pInfo->matchOne; @@ -684,7 +684,7 @@ static void likeFunc( sqlite3_value **argv ){ const unsigned char *zA, *zB; - int escape = 0; + u32 escape = 0; int nPat; sqlite3 *db = sqlite3_context_db_handle(context); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8cf8966fe0..83ddb2d5a4 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2879,7 +2879,7 @@ int sqlite3GetInt32(const char *, int*); int sqlite3Atoi(const char*); int sqlite3Utf16ByteLen(const void *pData, int nChar); int sqlite3Utf8CharLen(const char *pData, int nByte); -int sqlite3Utf8Read(const u8*, const u8**); +u32 sqlite3Utf8Read(const u8*, const u8**); /* ** Routines to read and write variable-length integers. These used to diff --git a/src/utf.c b/src/utf.c index 95182694d3..17f3a09a4f 100644 --- a/src/utf.c +++ b/src/utf.c @@ -163,7 +163,7 @@ static const unsigned char sqlite3Utf8Trans1[] = { || (c&0xFFFFF800)==0xD800 \ || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ } -int sqlite3Utf8Read( +u32 sqlite3Utf8Read( const unsigned char *zIn, /* First byte of UTF-8 character */ const unsigned char **pzNext /* Write first byte past UTF-8 char here */ ){ From 3fd651928f7fb142f07328e367ea33f2d92899c7 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Jun 2011 13:48:36 +0000 Subject: [PATCH 45/69] Changes to fts3auto.test to test OR, AND and NOT operations. FossilOrigin-Name: e4ab6cdb101bbeb804820425cf569ee7dc2397fb --- manifest | 12 +-- manifest.uuid | 2 +- test/fts3auto.test | 261 ++++++++++++++++++++++++++++++++++----------- 3 files changed, 205 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index c96551fae9..8bdca7e965 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sexposed\sby\scombining\smatchinfo(),\sNEAR\sand\s"ORDER\sBY\srowid\sDESC". -D 2011-06-13T09:11:01.953 +C Changes\sto\sfts3auto.test\sto\stest\sOR,\sAND\sand\sNOT\soperations. +D 2011-06-13T13:48:36.083 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test 696a2d32dd64a03aa47818c26ea64f8f27e7eb07 +F test/fts3auto.test 4aca0c0631fd40e79a45d8490828152b46ad4559 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 4e8dd19eef04777d800977faf1859a405e396f30 -R 6b4754b974de210b10ad796c5876a1bc +P 5f6b87f420f21749aa7c72e020c50aca74890086 +R ca51faeb6c7aef68bca6b7079ff7ebda U dan -Z f1c93614cce7c70636f14d06ee6a3496 +Z 9a71ae3656c163debcbb9f929d0e7d22 diff --git a/manifest.uuid b/manifest.uuid index 5c8deb0eac..45225befd7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f6b87f420f21749aa7c72e020c50aca74890086 \ No newline at end of file +e4ab6cdb101bbeb804820425cf569ee7dc2397fb \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index fe87d9f2c7..97b4304f0c 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -16,32 +16,13 @@ ifcapable !fts3 { finish_test ; return } source $testdir/fts3_common.tcl source $testdir/malloc_common.tcl -set testprefix fts3rnd2 +set testprefix fts3auto +set sfep $sqlite_fts3_enable_parentheses +set sqlite_fts3_enable_parentheses 1 -proc test_fts3_near_match {tn doc expr res} { - fts3_near_match $doc $expr -phrasecountvar p - uplevel do_test [list $tn] [list [list set {} $p]] [list $res] -} - -# Simple test cases for C routine [fts3_near_match]. +#-------------------------------------------------------------------------- +# Start of Tcl procs used by tests. # -test_fts3_near_match 1.1.1 {a b c a b} a {2} -test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c} {2 2 1} -test_fts3_near_match 1.1.3 {a b c a b} {"a b"} {2} -test_fts3_near_match 1.1.4 {a b c a b} {"b c"} {1} -test_fts3_near_match 1.1.5 {a b c a b} {"c c"} {0} - -test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f} {0 0} -test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f} {1 1} -test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b} {0 0} -test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b} {1 1} -test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0} -test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1} - -set A "a b c d e f g h i j k l m n o p q r s t u v w x y z" -test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0} -test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} - proc mit {blob} { set scan(littleEndian) i* set scan(bigEndian) I* @@ -62,24 +43,13 @@ proc fix_near_expr {expr} { return $out } -proc do_near_test {tn tbl expr} { +proc get_single_near_results {tbl expr arrayvar nullvar} { + upvar $arrayvar aMatchinfo + upvar $nullvar nullentry + catch {array unset aMatchinfo} set expr [fix_near_expr $expr] - # Create the MATCH expression from $expr - # - set match [lindex $expr 0] - if {[llength $match]>1} { - set match "\"$match\"" - } - foreach {nNear phrase} [lrange $expr 1 end] { - if {[llength $phrase]>1} { - append match " NEAR/$nNear \"$phrase\"" - } else { - append match " NEAR/$nNear $phrase" - } - } - # Calculate the expected results using [fts3_near_match]. The following # loop populates the "hits" and "counts" arrays as follows: # @@ -144,6 +114,116 @@ proc do_near_test {tn tbl expr} { set aMatchinfo($docid) $mi } + # Set up the nullentry output. + # + set nullentry [list] + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol<$nCol} {incr iCol} { + lappend nullentry 0 $nDoc($iPhrase,$iCol) $nHit($iPhrase,$iCol) + } + } +} + + +proc matching_brackets {expr} { + if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} { + return 0 + } + + set iBracket 1 + set nExpr [string length $expr] + for {set i 1} {$iBracket && $i < $nExpr} {incr i} { + set c [string range $expr $i $i] + if {$c == "("} {incr iBracket} + if {$c == ")"} {incr iBracket -1} + } + + return [expr ($iBracket==0 && $i==$nExpr)] +} + +proc get_near_results {tbl expr arrayvar {nullvar ""}} { + upvar $arrayvar aMatchinfo + if {$nullvar != ""} { upvar $nullvar nullentry } + + set expr [string trim $expr] + while { [matching_brackets $expr] } { + set expr [string trim [string range $expr 1 end-1]] + } + + set prec(NOT) 1 + set prec(AND) 2 + set prec(OR) 3 + + set currentprec 0 + set iBracket 0 + set expr_length [llength $expr] + for {set i 0} {$i < $expr_length} {incr i} { + set op [lindex $expr $i] + if {$iBracket==0 && [info exists prec($op)] && $prec($op)>=$currentprec } { + set opidx $i + set currentprec $prec($op) + } else { + for {set j 0} {$j < [string length $op]} {incr j} { + set c [string range $op $j $j] + if {$c == "("} { incr iBracket +1 } + if {$c == ")"} { incr iBracket -1 } + } + } + } + if {$iBracket!=0} { error "mismatched brackets in: $expr" } + + if {[info exists opidx]==0} { + get_single_near_results $tbl $expr aMatchinfo nullentry + } else { + set eLeft [lrange $expr 0 [expr $opidx-1]] + set eRight [lrange $expr [expr $opidx+1] end] + + get_near_results $tbl $eLeft aLeft nullleft + get_near_results $tbl $eRight aRight nullright + + switch -- [lindex $expr $opidx] { + "NOT" { + foreach hit [array names aLeft] { + if {0==[info exists aRight($hit)]} { + set aMatchinfo($hit) $aLeft($hit) + } + } + set nullentry $nullleft + } + + "AND" { + foreach hit [array names aLeft] { + if {[info exists aRight($hit)]} { + set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)] + } + } + set nullentry [concat $nullleft $nullright] + } + + "OR" { + foreach hit [array names aLeft] { + if {[info exists aRight($hit)]} { + set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)] + unset aRight($hit) + } else { + set aMatchinfo($hit) [concat $aLeft($hit) $nullright] + } + } + foreach hit [array names aRight] { + set aMatchinfo($hit) [concat $nullleft $aRight($hit)] + } + + set nullentry [concat $nullleft $nullright] + } + } + } +} + +proc do_near_test {tn tbl expr} { + + get_near_results $tbl $expr aMatchinfo + set match $expr + set matchinfo_asc [list] foreach docid [lsort -integer -incr [array names aMatchinfo]] { lappend matchinfo_asc $docid $aMatchinfo($docid) @@ -153,15 +233,15 @@ proc do_near_test {tn tbl expr} { lappend matchinfo_desc $docid $aMatchinfo($docid) } - set title "(\"$match\" -> [llength [array names hits]] rows)" + set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)" do_execsql_test $tn$title.1 " SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC - " [lsort -integer -incr [array names hits]] + " [lsort -integer -incr [array names aMatchinfo]] do_execsql_test $tn$title.2 " SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC - " [lsort -integer -decr [array names hits]] + " [lsort -integer -decr [array names aMatchinfo]] do_execsql_test $tn$title.3 " SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl @@ -174,28 +254,83 @@ proc do_near_test {tn tbl expr} { " $matchinfo_asc } -do_test 2.1 { - execsql { CREATE VIRTUAL TABLE t1 USING fts3(a, b) } - for {set i 0} {$i<32} {incr i} { - set doc [list] - if {$i&0x01} {lappend doc one} - if {$i&0x02} {lappend doc two} - if {$i&0x04} {lappend doc three} - if {$i&0x08} {lappend doc four} - if {$i&0x10} {lappend doc five} - execsql { INSERT INTO t1 VALUES($doc, null) } - } -} {} -foreach {tn expr} { - 1 {one} - 2 {one NEAR/1 five} - 3 {t*} - 4 {t* NEAR/0 five} - 5 {o* NEAR/1 f*} - 6 {one NEAR five NEAR two NEAR four NEAR three} -} { - do_near_test 2.2.$tn t1 $expr + +# End of test procs. Actual tests are below this line. +#-------------------------------------------------------------------------- + +#-------------------------------------------------------------------------- +# The following test cases - fts3auto-1.* - focus on testing the Tcl +# command [fts3_near_match], which is used by other tests in this file. +# +proc test_fts3_near_match {tn doc expr res} { + fts3_near_match $doc $expr -phrasecountvar p + uplevel do_test [list $tn] [list [list set {} $p]] [list $res] } +test_fts3_near_match 1.1.1 {a b c a b} a {2} +test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c} {2 2 1} +test_fts3_near_match 1.1.3 {a b c a b} {"a b"} {2} +test_fts3_near_match 1.1.4 {a b c a b} {"b c"} {1} +test_fts3_near_match 1.1.5 {a b c a b} {"c c"} {0} + +test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f} {0 0} +test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f} {1 1} +test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b} {0 0} +test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b} {1 1} +test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0} +test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1} + +set A "a b c d e f g h i j k l m n o p q r s t u v w x y z" +test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0} +test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} + +#-------------------------------------------------------------------------- +# Test cases fts3auto-2.* run some simple tests using the +# [do_near_test] proc. +# +foreach {tn create} { + 1 "CREATE VIRTUAL TABLE t1 USING fts4(a, b)" + 2 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC)" + 3 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC)" + 4 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, prefix=1)" + 5 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC, prefix=1)" + 6 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC, prefix=1)" +} { + do_test 2.$tn.1 { + catchsql { DROP TABLE t1 } + execsql $create + for {set i 0} {$i<32} {incr i} { + set doc [list] + if {$i&0x01} {lappend doc one} + if {$i&0x02} {lappend doc two} + if {$i&0x04} {lappend doc three} + if {$i&0x08} {lappend doc four} + if {$i&0x10} {lappend doc five} + execsql { INSERT INTO t1 VALUES($doc, null) } + } + } {} + foreach {tn2 expr} { + 1 {one} + 2 {one NEAR/1 five} + 3 {t*} + 4 {t* NEAR/0 five} + 5 {o* NEAR/1 f*} + 6 {one NEAR five NEAR two NEAR four NEAR three} + 7 {one NEAR xyz} + 8 {one OR two} + 9 {one AND two} + 10 {one NOT two} + 11 {one AND two OR three} + 12 {three OR one AND two} + 13 {(three OR one) AND two} + 14 {(three OR one) AND two NOT (five NOT four)} + 15 {"one two"} + 16 {"one two" NOT "three four"} + } { + do_near_test 2.$tn.2.$tn2 t1 $expr + } +} + +set sqlite_fts3_enable_parentheses $sfep finish_test From 2f779343854941321b64995cd608e6fe7346ad2d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Jun 2011 17:00:12 +0000 Subject: [PATCH 46/69] Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and deferred tokens. FossilOrigin-Name: b9fb69e55bb05a8819688ee63987f2a05c33d59b --- ext/fts3/fts3.c | 13 +++-- manifest | 16 +++--- manifest.uuid | 2 +- test/fts3auto.test | 125 ++++++++++++++++++++++++++++++++++++++----- test/fts3defer2.test | 5 +- 5 files changed, 131 insertions(+), 30 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 4b37c4e645..17a7b2ccaa 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3572,11 +3572,16 @@ static void fts3EvalTokenCosts( } }else if( pExpr->eType!=FTSQUERY_NOT ){ if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr; - **ppOr = pExpr; + pRoot = pExpr->pLeft; + **ppOr = pRoot; (*ppOr)++; } fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); } } @@ -3736,7 +3741,7 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( sizeof(Fts3TokenAndCost) * nToken - + sizeof(Fts3Expr *) * nOr + + sizeof(Fts3Expr *) * nOr * 2 ); apOr = (Fts3Expr **)&aTC[nToken]; @@ -4310,7 +4315,7 @@ int sqlite3Fts3EvalPhraseStats( int rc = SQLITE_OK; int iCol; - if( pExpr->bDeferred ){ + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ assert( pCsr->nDoc>0 ); for(iCol=0; iColnColumn; iCol++){ aiOut[iCol*3 + 1] = pCsr->nDoc; diff --git a/manifest b/manifest index 8bdca7e965..15429e6135 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sfts3auto.test\sto\stest\sOR,\sAND\sand\sNOT\soperations. -D 2011-06-13T13:48:36.083 +C Add\stests\sfor\sdeferred\stokens\sto\sfts3auto.test.\sFix\sa\sproblem\swith\sOR\squeries\sand\sdeferred\stokens. +D 2011-06-13T17:00:12.630 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c e71dafb1f324358d12fd02ea12644d8c6cea577a +F ext/fts3/fts3.c 2a48945d199d5d0fe66a06c8cf73ad2f0e7a4b5f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test 4aca0c0631fd40e79a45d8490828152b46ad4559 +F test/fts3auto.test bf01a1d793155d744d23e4092871ca116f1db64c F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -466,7 +466,7 @@ F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35 -F test/fts3defer2.test ad5bd3ae616fb719eb06b0a73eb5c8fb1c411989 +F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 5f6b87f420f21749aa7c72e020c50aca74890086 -R ca51faeb6c7aef68bca6b7079ff7ebda +P e4ab6cdb101bbeb804820425cf569ee7dc2397fb +R 7fcdbf104822a42776783c2cd9e89101 U dan -Z 9a71ae3656c163debcbb9f929d0e7d22 +Z 0999cf6b26be67f673d9db0918d86701 diff --git a/manifest.uuid b/manifest.uuid index 45225befd7..41624e6daa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4ab6cdb101bbeb804820425cf569ee7dc2397fb \ No newline at end of file +b9fb69e55bb05a8819688ee63987f2a05c33d59b \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 97b4304f0c..68c82c8a85 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -93,12 +93,45 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { set iCol [lindex [split $key ,] 1] set iPhrase 0 foreach c $counts($key) { - if {$c>0} { incr nHit($iPhrase,$iCol) 1 } - incr nDoc($iPhrase,$iCol) $c + if {$c>0} { incr nDoc($iPhrase,$iCol) 1 } + incr nHit($iPhrase,$iCol) $c incr iPhrase } } + if {[info exists ::fts3_deferred] && [llength $expr]==1} { + set phrase [lindex $expr 0] + set rewritten [list] + set partial 0 + foreach tok $phrase { + if {[lsearch $::fts3_deferred $tok]>=0} { + lappend rewritten * + } else { + lappend rewritten $tok + set partial 1 + } + } + if {$partial==0} { + set tblsize [db one "SELECT count(*) FROM $tbl"] + for {set iCol 0} {$iCol < $nCol} {incr iCol} { + set nHit(0,$iCol) $tblsize + set nDoc(0,$iCol) $tblsize + } + } elseif {$rewritten != $phrase} { + while {[lindex $rewritten end] == "*"} { + set rewritten [lrange $rewritten 0 end-1] + } + while {[lindex $rewritten 0] == "*"} { + set rewritten [lrange $rewritten 1 end] + } + get_single_near_results $tbl [list $rewritten] aRewrite nullentry + foreach docid [array names hits] { + set aMatchinfo($docid) $aRewrite($docid) + } + return + } + } + # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to # contain the output of matchinfo('x') for the document. # @@ -107,8 +140,8 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { for {set iCol 0} {$iCol<$nCol} {incr iCol} { lappend mi [lindex $counts($docid,$iCol) $iPhrase] - lappend mi $nDoc($iPhrase,$iCol) lappend mi $nHit($iPhrase,$iCol) + lappend mi $nDoc($iPhrase,$iCol) } } set aMatchinfo($docid) $mi @@ -119,7 +152,7 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { set nullentry [list] for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { for {set iCol 0} {$iCol<$nCol} {incr iCol} { - lappend nullentry 0 $nDoc($iPhrase,$iCol) $nHit($iPhrase,$iCol) + lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol) } } } @@ -219,7 +252,7 @@ proc get_near_results {tbl expr arrayvar {nullvar ""}} { } } -proc do_near_test {tn tbl expr} { +proc do_fts3query_test {tn tbl expr} { get_near_results $tbl $expr aMatchinfo set match $expr @@ -286,19 +319,20 @@ test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} #-------------------------------------------------------------------------- # Test cases fts3auto-2.* run some simple tests using the -# [do_near_test] proc. +# [do_fts3query_test] proc. # foreach {tn create} { - 1 "CREATE VIRTUAL TABLE t1 USING fts4(a, b)" - 2 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC)" - 3 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC)" - 4 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, prefix=1)" - 5 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC, prefix=1)" - 6 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC, prefix=1)" + 1 "fts4(a, b)" + 2 "fts4(a, b, order=DESC)" + 3 "fts4(a, b, order=ASC)" + 4 "fts4(a, b, prefix=1)" + 5 "fts4(a, b, order=DESC, prefix=1)" + 6 "fts4(a, b, order=ASC, prefix=1)" } { +break do_test 2.$tn.1 { catchsql { DROP TABLE t1 } - execsql $create + execsql "CREATE VIRTUAL TABLE t1 USING $create" for {set i 0} {$i<32} {incr i} { set doc [list] if {$i&0x01} {lappend doc one} @@ -309,6 +343,7 @@ foreach {tn create} { execsql { INSERT INTO t1 VALUES($doc, null) } } } {} + foreach {tn2 expr} { 1 {one} 2 {one NEAR/1 five} @@ -327,7 +362,69 @@ foreach {tn create} { 15 {"one two"} 16 {"one two" NOT "three four"} } { - do_near_test 2.$tn.2.$tn2 t1 $expr + do_fts3query_test 2.$tn.2.$tn2 t1 $expr + } +} + +#-------------------------------------------------------------------------- +# Some test cases involving deferred tokens. +# +proc make_token_deferrable {tbl token} { + set nRow [db one "SELECT count(*) FROM $tbl"] + set pgsz [db one "PRAGMA page_size"] + + execsql "INSERT INTO $tbl ($tbl) VALUES('maxpending=100000000')" + execsql BEGIN + for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { + set doc [string repeat "$token " 100] + execsql "INSERT INTO $tbl VALUES(\$doc)" + } + execsql "INSERT INTO $tbl VALUES('aaaaaaa ${token}aaaaa')" + execsql COMMIT +} + +foreach {tn create} { + 1 "fts4(x)" + 2 "fts4(x, order=DESC)" +} { + catchsql { DROP TABLE t1 } + execsql "CREATE VIRTUAL TABLE t1 USING $create" + do_execsql_test 3.$tn.1 { + INSERT INTO t1 VALUES('a b c d e f g h i j k'); + INSERT INTO t1 VALUES('b c d e f g h i j k a'); + INSERT INTO t1 VALUES('c d e f g h i j k a b'); + INSERT INTO t1 VALUES('d e f g h i j k a b c'); + INSERT INTO t1 VALUES('e f g h i j k a b c d'); + INSERT INTO t1 VALUES('f g h i j k a b c d e'); + INSERT INTO t1 VALUES('a c e g i k'); + INSERT INTO t1 VALUES('a d g j'); + INSERT INTO t1 VALUES('c a b'); + } + + make_token_deferrable t1 c + + set ::fts3_deferred [list] + foreach {tn2 expr} { + 1 {a OR c} + } { + do_fts3query_test 3.$tn.2.$tn2 t1 $expr + } + + set ::fts3_deferred [list c] + execsql { + UPDATE t1_segments + SET block = zeroblob(length(block)) + WHERE length(block)>10000 AND 0 + } + foreach {tn2 expr} { + 1 {a NEAR c} + 2 {a AND c} + 3 {"a c"} + 4 {"c a"} + 5 {"a c" NEAR/1 g} + 6 {"a c" NEAR/0 g} + } { + do_fts3query_test 3.$tn.2.$tn2 t1 $expr } } diff --git a/test/fts3defer2.test b/test/fts3defer2.test index b7bf923550..92a4491efd 100644 --- a/test/fts3defer2.test +++ b/test/fts3defer2.test @@ -56,14 +56,13 @@ do_execsql_test 1.2.1 { SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } {{a b c d e f a x y}} -breakpoint do_execsql_test 1.2.2 { SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } [list \ {a b c d [e] [f] [a] x y} \ {0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 1 1 1 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 1 1 1 8 5001 9] ] do_execsql_test 1.2.3 { @@ -72,7 +71,7 @@ do_execsql_test 1.2.3 { } [list \ {[a] b c d [e] [f] [a] x y} \ {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 1 1 2 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 2 2 1 8 5001 9] ] do_execsql_test 1.3.1 { DROP TABLE t1 } From 653265d9ed4a3eb1d088147dd0e750821e49b857 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Jun 2011 18:21:11 +0000 Subject: [PATCH 47/69] Update trace2.test to account for new FTS queries. FossilOrigin-Name: 2c20129297b64f4113b8edb551385eb918279471 --- manifest | 14 +++--- manifest.uuid | 2 +- test/fts3auto.test | 110 ++++++++++++++++++++++++++------------------- test/trace2.test | 6 +-- 4 files changed, 75 insertions(+), 57 deletions(-) diff --git a/manifest b/manifest index 15429e6135..13395c5a81 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sdeferred\stokens\sto\sfts3auto.test.\sFix\sa\sproblem\swith\sOR\squeries\sand\sdeferred\stokens. -D 2011-06-13T17:00:12.630 +C Update\strace2.test\sto\saccount\sfor\snew\sFTS\squeries. +D 2011-06-13T18:21:11.056 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test bf01a1d793155d744d23e4092871ca116f1db64c +F test/fts3auto.test 0173f3fc3e03d66beb6dd65f7b99e7eb4ae2a27d F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -821,7 +821,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 -F test/trace2.test 0ce11265c83333d8f5beeca19e71ed93a88d386c +F test/trace2.test 962175290996d5f06dc4402ca218bbfc7df4cb20 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test d728abaa318ca364dc370e06576aa7e5fbed7e97 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P e4ab6cdb101bbeb804820425cf569ee7dc2397fb -R 7fcdbf104822a42776783c2cd9e89101 +P b9fb69e55bb05a8819688ee63987f2a05c33d59b +R 4f421cd82a93c9e8973177e58e4c866f U dan -Z 0999cf6b26be67f673d9db0918d86701 +Z 2a2f406b9fac55b2e6e57d2ed45fc4da diff --git a/manifest.uuid b/manifest.uuid index 41624e6daa..5186fdc744 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9fb69e55bb05a8819688ee63987f2a05c33d59b \ No newline at end of file +2c20129297b64f4113b8edb551385eb918279471 \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 68c82c8a85..f8f1e77b42 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -21,8 +21,62 @@ set sfep $sqlite_fts3_enable_parentheses set sqlite_fts3_enable_parentheses 1 #-------------------------------------------------------------------------- -# Start of Tcl procs used by tests. +# Start of Tcl infrastructure used by tests. The entry point is +# [do_fts3query_test] (described below). # + +# do_fts3query_test TESTNAME ?OPTIONS? TABLE MATCHEXPR +# +# This proc runs several test cases on FTS3/4 table $TABLE using match +# expression $MATCHEXPR. All documents in $TABLE must be formatted so that +# they can be "tokenized" using the Tcl list commands (llength, lindex etc.). +# The name and column names used by $TABLE must not require any quoting or +# escaping when used in SQL statements. +# +# $MATCHINFO may be any expression accepted by the FTS4 MATCH operator, +# except that the ":token" syntax is not supported. Tcl list +# commands are used to tokenize the expression. Any parenthesis must appear +# either as separate list elements, or as the first (for opening) or last +# (for closing) character of a list element. i.e. the expression "(a OR b)c" +# will not be parsed correctly, but "( a OR b) c" will. +# +set sqlite_fts3_enable_parentheses 1 +proc do_fts3query_test {tn tbl expr} { + + get_near_results $tbl $expr aMatchinfo + set match $expr + + set matchinfo_asc [list] + foreach docid [lsort -integer -incr [array names aMatchinfo]] { + lappend matchinfo_asc $docid $aMatchinfo($docid) + } + set matchinfo_desc [list] + foreach docid [lsort -integer -decr [array names aMatchinfo]] { + lappend matchinfo_desc $docid $aMatchinfo($docid) + } + + set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)" + + do_execsql_test $tn$title.1 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC + " [lsort -integer -incr [array names aMatchinfo]] + + do_execsql_test $tn$title.2 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC + " [lsort -integer -decr [array names aMatchinfo]] + + do_execsql_test $tn$title.3 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid DESC + " $matchinfo_desc + + do_execsql_test $tn$title.4 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid ASC + " $matchinfo_asc +} + + proc mit {blob} { set scan(littleEndian) i* set scan(bigEndian) I* @@ -252,41 +306,6 @@ proc get_near_results {tbl expr arrayvar {nullvar ""}} { } } -proc do_fts3query_test {tn tbl expr} { - - get_near_results $tbl $expr aMatchinfo - set match $expr - - set matchinfo_asc [list] - foreach docid [lsort -integer -incr [array names aMatchinfo]] { - lappend matchinfo_asc $docid $aMatchinfo($docid) - } - set matchinfo_desc [list] - foreach docid [lsort -integer -decr [array names aMatchinfo]] { - lappend matchinfo_desc $docid $aMatchinfo($docid) - } - - set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)" - - do_execsql_test $tn$title.1 " - SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC - " [lsort -integer -incr [array names aMatchinfo]] - - do_execsql_test $tn$title.2 " - SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC - " [lsort -integer -decr [array names aMatchinfo]] - - do_execsql_test $tn$title.3 " - SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl - WHERE $tbl MATCH '$match' ORDER BY docid DESC - " $matchinfo_desc - - do_execsql_test $tn$title.4 " - SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl - WHERE $tbl MATCH '$match' ORDER BY docid ASC - " $matchinfo_asc -} - # End of test procs. Actual tests are below this line. #-------------------------------------------------------------------------- @@ -329,7 +348,6 @@ foreach {tn create} { 5 "fts4(a, b, order=DESC, prefix=1)" 6 "fts4(a, b, order=ASC, prefix=1)" } { -break do_test 2.$tn.1 { catchsql { DROP TABLE t1 } execsql "CREATE VIRTUAL TABLE t1 USING $create" @@ -390,15 +408,15 @@ foreach {tn create} { catchsql { DROP TABLE t1 } execsql "CREATE VIRTUAL TABLE t1 USING $create" do_execsql_test 3.$tn.1 { - INSERT INTO t1 VALUES('a b c d e f g h i j k'); - INSERT INTO t1 VALUES('b c d e f g h i j k a'); - INSERT INTO t1 VALUES('c d e f g h i j k a b'); - INSERT INTO t1 VALUES('d e f g h i j k a b c'); - INSERT INTO t1 VALUES('e f g h i j k a b c d'); - INSERT INTO t1 VALUES('f g h i j k a b c d e'); - INSERT INTO t1 VALUES('a c e g i k'); - INSERT INTO t1 VALUES('a d g j'); - INSERT INTO t1 VALUES('c a b'); + INSERT INTO t1(docid, x) VALUES(-2, 'a b c d e f g h i j k'); + INSERT INTO t1(docid, x) VALUES(-1, 'b c d e f g h i j k a'); + INSERT INTO t1(docid, x) VALUES(0, 'c d e f g h i j k a b'); + INSERT INTO t1(docid, x) VALUES(1, 'd e f g h i j k a b c'); + INSERT INTO t1(docid, x) VALUES(2, 'e f g h i j k a b c d'); + INSERT INTO t1(docid, x) VALUES(3, 'f g h i j k a b c d e'); + INSERT INTO t1(docid, x) VALUES(4, 'a c e g i k'); + INSERT INTO t1(docid, x) VALUES(5, 'a d g j'); + INSERT INTO t1(docid, x) VALUES(6, 'c a b'); } make_token_deferrable t1 c diff --git a/test/trace2.test b/test/trace2.test index a70dd89d1d..42738db3aa 100644 --- a/test/trace2.test +++ b/test/trace2.test @@ -141,10 +141,10 @@ ifcapable fts3 { INSERT INTO x1(x1) VALUES('optimize'); } { "INSERT INTO x1(x1) VALUES('optimize');" - "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' ORDER BY level DESC, idx ASC" - "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'" + "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?ORDER BY level DESC, idx ASC" + "-- SELECT max(level) FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" - "-- DELETE FROM 'main'.'x1_segdir'" + "-- DELETE FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?" "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } } From 806cbfc2149da1de634c9fcf685428b732a57ca8 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Jun 2011 07:14:43 +0000 Subject: [PATCH 48/69] Remove unused parameters from internal fts3 function. FossilOrigin-Name: 06de3f2cbc27cdfd9f83218c9ea576f74f60d07b --- ext/fts3/fts3.c | 54 ++++++++++++++++++++++++------------------------- manifest | 12 +++++------ manifest.uuid | 2 +- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 17a7b2ccaa..60b8595e84 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3342,21 +3342,6 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ return SQLITE_OK; } - -/* -** The following three functions: -** -** fts3EvalPhraseStart() -** fts3EvalPhraseNext() -** -** May be used with a phrase object after fts3EvalAllocateReaders() has been -** called to iterate through the set of docids that match the phrase. -** -** After a successful call to fts3EvalPhraseNext(), the following two -** functions may be called to access the current docid and position-list. -*/ - - /* ** This function is called for each Fts3Phrase in a full-text query ** expression to initialize the mechanism for returning rows. Once this @@ -4158,12 +4143,16 @@ static void fts3EvalRestart( } } -static void fts3EvalUpdateCounts( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int *pRc -){ - if( pExpr && *pRc==SQLITE_OK ){ +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ + if( pExpr ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase && pPhrase->doclist.pList ){ int iCol = 0; @@ -4189,14 +4178,25 @@ static void fts3EvalUpdateCounts( } } - fts3EvalUpdateCounts(pCsr, pExpr->pLeft, pRc); - fts3EvalUpdateCounts(pCsr, pExpr->pRight, pRc); + fts3EvalUpdateCounts(pExpr->pLeft); + fts3EvalUpdateCounts(pExpr->pRight); } } +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ static int fts3EvalGatherStats( - Fts3Cursor *pCsr, - Fts3Expr *pExpr + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ ){ int rc = SQLITE_OK; /* Return code */ @@ -4248,8 +4248,8 @@ static int fts3EvalGatherStats( && fts3EvalLoadDeferred(pCsr, &rc) ); - if( pCsr->isEof==0 ){ - fts3EvalUpdateCounts(pCsr, pRoot, &rc); + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot); } } diff --git a/manifest b/manifest index 13395c5a81..037d939c2d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\strace2.test\sto\saccount\sfor\snew\sFTS\squeries. -D 2011-06-13T18:21:11.056 +C Remove\sunused\sparameters\sfrom\sinternal\sfts3\sfunction. +D 2011-06-14T07:14:43.149 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 2a48945d199d5d0fe66a06c8cf73ad2f0e7a4b5f +F ext/fts3/fts3.c f4f65273121386349993d600c4c5f710d4b3e956 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P b9fb69e55bb05a8819688ee63987f2a05c33d59b -R 4f421cd82a93c9e8973177e58e4c866f +P 2c20129297b64f4113b8edb551385eb918279471 +R 24738ea4e4f82770b7e9961a6f31bf0f U dan -Z 2a2f406b9fac55b2e6e57d2ed45fc4da +Z 096f5065b2f90702df92a748235d8620 diff --git a/manifest.uuid b/manifest.uuid index 5186fdc744..6cb1ded47e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c20129297b64f4113b8edb551385eb918279471 \ No newline at end of file +06de3f2cbc27cdfd9f83218c9ea576f74f60d07b \ No newline at end of file From 2cf1a1de4bad0aa492f94d525c68b524d394b5a7 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Jun 2011 09:00:27 +0000 Subject: [PATCH 49/69] Fix another bug caused by NEAR/matchinfo/order=DESC interaction. FossilOrigin-Name: 04907fbadeb743c95cc9f3529e63ef388684799f --- ext/fts3/fts3.c | 7 ++- manifest | 16 +++---- manifest.uuid | 2 +- test/fts3auto.test | 97 ++++++++++++++++++++++++++++++++---------- test/permutations.test | 2 +- 5 files changed, 87 insertions(+), 37 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index e057eb3825..feed5220c9 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3387,7 +3387,6 @@ void sqlite3Fts3DoclistPrev( u8 *pbEof /* OUT: End-of-file flag */ ){ char *p = *ppIter; - int iMul = (bDescIdx ? -1 : 1); assert( nDoclist>0 ); assert( *pbEof==0 ); @@ -3399,10 +3398,8 @@ void sqlite3Fts3DoclistPrev( char *pNext = 0; char *pDocid = aDoclist; char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; - pDocid += sqlite3Fts3GetVarint(pDocid, &iDocid); - pNext = pDocid; - fts3PoslistCopy(0, &pDocid); while( pDocid=0} { + if {[lsearch $deferred $tok]>=0} { lappend rewritten * } else { lappend rewritten $tok @@ -178,7 +205,7 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { while {[lindex $rewritten 0] == "*"} { set rewritten [lrange $rewritten 1 end] } - get_single_near_results $tbl [list $rewritten] aRewrite nullentry + get_single_near_results $tbl [list $rewritten] {} aRewrite nullentry foreach docid [array names hits] { set aMatchinfo($docid) $aRewrite($docid) } @@ -228,7 +255,7 @@ proc matching_brackets {expr} { return [expr ($iBracket==0 && $i==$nExpr)] } -proc get_near_results {tbl expr arrayvar {nullvar ""}} { +proc get_near_results {tbl expr deferred arrayvar {nullvar ""}} { upvar $arrayvar aMatchinfo if {$nullvar != ""} { upvar $nullvar nullentry } @@ -260,13 +287,13 @@ proc get_near_results {tbl expr arrayvar {nullvar ""}} { if {$iBracket!=0} { error "mismatched brackets in: $expr" } if {[info exists opidx]==0} { - get_single_near_results $tbl $expr aMatchinfo nullentry + get_single_near_results $tbl $expr $deferred aMatchinfo nullentry } else { set eLeft [lrange $expr 0 [expr $opidx-1]] set eRight [lrange $expr [expr $opidx+1] end] - get_near_results $tbl $eLeft aLeft nullleft - get_near_results $tbl $eRight aRight nullright + get_near_results $tbl $eLeft $deferred aLeft nullleft + get_near_results $tbl $eRight $deferred aRight nullright switch -- [lindex $expr $opidx] { "NOT" { @@ -390,8 +417,6 @@ foreach {tn create} { proc make_token_deferrable {tbl token} { set nRow [db one "SELECT count(*) FROM $tbl"] set pgsz [db one "PRAGMA page_size"] - - execsql "INSERT INTO $tbl ($tbl) VALUES('maxpending=100000000')" execsql BEGIN for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { set doc [string repeat "$token " 100] @@ -421,28 +446,54 @@ foreach {tn create} { make_token_deferrable t1 c - set ::fts3_deferred [list] foreach {tn2 expr} { 1 {a OR c} } { do_fts3query_test 3.$tn.2.$tn2 t1 $expr } - set ::fts3_deferred [list c] execsql { UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000 AND 0 } - foreach {tn2 expr} { - 1 {a NEAR c} - 2 {a AND c} - 3 {"a c"} - 4 {"c a"} - 5 {"a c" NEAR/1 g} - 6 {"a c" NEAR/0 g} + foreach {tn2 expr def} { + 1 {a NEAR c} {} + 2 {a AND c} c + 3 {"a c"} c + 4 {"c a"} c + 5 {"a c" NEAR/1 g} {} + 6 {"a c" NEAR/0 g} {} } { - do_fts3query_test 3.$tn.2.$tn2 t1 $expr + do_fts3query_test 3.$tn.2.$tn2 -deferred $def t1 $expr + } +} + +#-------------------------------------------------------------------------- +# +foreach {tn create} { + 1 "fts4(x, y)" + 2 "fts4(x, y, order=DESC)" +} { + catchsql { DROP TABLE t1 } + execsql "CREATE VIRTUAL TABLE t1 USING $create" + + foreach {x y} { + {one two five four five} {} + {} {one two five four five} + {one two} {five four five} + } { + execsql {INSERT INTO t1 VALUES($x, $y)} + } + + foreach {tn2 expr} { + 1 {one AND five} + 2 {one NEAR five} + 3 {one NEAR/1 five} + 4 {one NEAR/2 five} + 5 {one NEAR/3 five} + } { + do_fts3query_test 4.$tn.2.$tn2 t1 $expr } } diff --git a/test/permutations.test b/test/permutations.test index 4640ed1139..9c95296e39 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -183,7 +183,7 @@ test_suite "fts3" -prefix "" -description { fts3fault.test fts3malloc.test fts3matchinfo.test - fts3aux1.test fts3comp1.test + fts3aux1.test fts3comp1.test fts3auto.test } From db27fc078534611a934bafad541f2157c2ba4507 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Jun 2011 11:32:50 +0000 Subject: [PATCH 50/69] Add a couple of extra tests. FossilOrigin-Name: aefd46dfae7e06fbaf4f2b9a86a7f2ac6927331e --- manifest | 12 ++--- manifest.uuid | 2 +- test/fts3auto.test | 121 ++++++++++++++++++++++++++++----------------- 3 files changed, 84 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 252014ea4d..bc0807804f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sbug\scaused\sby\sNEAR/matchinfo/order=DESC\sinteraction. -D 2011-06-14T09:00:27.140 +C Add\sa\scouple\sof\sextra\stests. +D 2011-06-14T11:32:50.766 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test 337dff2758887bfe7d337f76d25cc48db6e26f28 +F test/fts3auto.test 2f86f2a0e8ffa26d81d570897e6cc1c2262256d5 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 135ce30f62ebd6a1b239c18dbbd9c926ea507db4 -R 6437acdf1435a01de33437d42862b03a +P 04907fbadeb743c95cc9f3529e63ef388684799f +R 84da75d7a3a06308ee488ec7371657d8 U dan -Z 0ea30b6b1ae90b1e541e7d3c3cb40fc6 +Z 0b29b5535d6bfbaa65e9d19cac80576d diff --git a/manifest.uuid b/manifest.uuid index 79eee6ffa9..3ddb9fa573 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04907fbadeb743c95cc9f3529e63ef388684799f \ No newline at end of file +aefd46dfae7e06fbaf4f2b9a86a7f2ac6927331e \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 47429308a0..5c9b85128b 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -21,10 +21,14 @@ set sfep $sqlite_fts3_enable_parentheses set sqlite_fts3_enable_parentheses 1 #-------------------------------------------------------------------------- -# Start of Tcl infrastructure used by tests. The entry point is -# [do_fts3query_test] (described below). +# Start of Tcl infrastructure used by tests. The entry points are: +# +# do_fts3query_test +# fts3_make_deferrable +# fts3_zero_long_segments # +# # do_fts3query_test TESTNAME ?OPTIONS? TABLE MATCHEXPR # # This proc runs several test cases on FTS3/4 table $TABLE using match @@ -103,6 +107,38 @@ proc do_fts3query_test {tn args} { " $matchinfo_asc } +# fts3_make_deferrable TABLE TOKEN +# +proc fts3_make_deferrable {tbl token} { + + set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy] + set name [sqlite3_column_name $stmt 0] + sqlite3_finalize $stmt + + set nRow [db one "SELECT count(*) FROM $tbl"] + set pgsz [db one "PRAGMA page_size"] + execsql BEGIN + for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { + set doc [string repeat "$token " 100] + execsql "INSERT INTO $tbl ($name) VALUES(\$doc)" + } + execsql "INSERT INTO $tbl ($name) VALUES('aaaaaaa ${token}aaaaa')" + execsql COMMIT + + return [expr $nRow*$pgsz] +} + +# fts3_zero_long_segments TABLE ?LIMIT? +# +proc fts3_zero_long_segments {tbl limit} { + execsql " + UPDATE ${tbl}_segments + SET block = zeroblob(length(block)) + WHERE length(block)>$limit + " + return [db changes] +} + proc mit {blob} { set scan(littleEndian) i* @@ -414,17 +450,6 @@ foreach {tn create} { #-------------------------------------------------------------------------- # Some test cases involving deferred tokens. # -proc make_token_deferrable {tbl token} { - set nRow [db one "SELECT count(*) FROM $tbl"] - set pgsz [db one "PRAGMA page_size"] - execsql BEGIN - for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { - set doc [string repeat "$token " 100] - execsql "INSERT INTO $tbl VALUES(\$doc)" - } - execsql "INSERT INTO $tbl VALUES('aaaaaaa ${token}aaaaa')" - execsql COMMIT -} foreach {tn create} { 1 "fts4(x)" @@ -444,19 +469,14 @@ foreach {tn create} { INSERT INTO t1(docid, x) VALUES(6, 'c a b'); } - make_token_deferrable t1 c + set limit [fts3_make_deferrable t1 c] - foreach {tn2 expr} { - 1 {a OR c} - } { - do_fts3query_test 3.$tn.2.$tn2 t1 $expr - } + do_fts3query_test 3.$tn.2.1 t1 {a OR c} + + do_test 3.$tn.3 { + fts3_zero_long_segments t1 $limit + } {1} - execsql { - UPDATE t1_segments - SET block = zeroblob(length(block)) - WHERE length(block)>10000 AND 0 - } foreach {tn2 expr def} { 1 {a NEAR c} {} 2 {a AND c} c @@ -465,36 +485,49 @@ foreach {tn create} { 5 {"a c" NEAR/1 g} {} 6 {"a c" NEAR/0 g} {} } { - do_fts3query_test 3.$tn.2.$tn2 -deferred $def t1 $expr + do_fts3query_test 3.$tn.4.$tn2 -deferred $def t1 $expr } } #-------------------------------------------------------------------------- -# +# foreach {tn create} { 1 "fts4(x, y)" 2 "fts4(x, y, order=DESC)" + 3 "fts4(x, y, order=DESC, prefix=2)" } { - catchsql { DROP TABLE t1 } - execsql "CREATE VIRTUAL TABLE t1 USING $create" - foreach {x y} { - {one two five four five} {} - {} {one two five four five} - {one two} {five four five} - } { - execsql {INSERT INTO t1 VALUES($x, $y)} - } + execsql [subst { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING $create; + INSERT INTO t1 VALUES('one two five four five', ''); + INSERT INTO t1 VALUES('', 'one two five four five'); + INSERT INTO t1 VALUES('one two', 'five four five'); + }] - foreach {tn2 expr} { - 1 {one AND five} - 2 {one NEAR five} - 3 {one NEAR/1 five} - 4 {one NEAR/2 five} - 5 {one NEAR/3 five} - } { - do_fts3query_test 4.$tn.2.$tn2 t1 $expr - } + do_fts3query_test 4.$tn.1.1 t1 {one AND five} + do_fts3query_test 4.$tn.1.2 t1 {one NEAR five} + do_fts3query_test 4.$tn.1.3 t1 {one NEAR/1 five} + do_fts3query_test 4.$tn.1.4 t1 {one NEAR/2 five} + do_fts3query_test 4.$tn.1.5 t1 {one NEAR/3 five} + + do_test 4.$tn.2 { + set limit [fts3_make_deferrable t1 five] + execsql { INSERT INTO t1(t1) VALUES('optimize') } + expr {[fts3_zero_long_segments t1 $limit]>0} + } {1} + + do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five} + do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five} + do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five} + do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five} + do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five} + + do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*} + do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*} + do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*} + do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*} + do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*} } set sqlite_fts3_enable_parentheses $sfep From 5f84e14add49456366e05fc606a9e329b1e04d5a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Jun 2011 14:18:45 +0000 Subject: [PATCH 51/69] Fix a memory leak that can follow an OOM error in a user-function that uses sqlite3_set_auxdata(). FossilOrigin-Name: 0185c4b689d18d66e6aa39b4a7bddc279e3c9d17 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 19 ++++++++++--------- test/malloc.test | 12 ++++++++++++ 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 3f4c162a0d..54f991e432 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfts3-prefix-search\sbranch\swith\strunk. -D 2011-06-14T11:50:09.808 +C Fix\sa\smemory\sleak\sthat\scan\sfollow\san\sOOM\serror\sin\sa\suser-function\sthat\suses\ssqlite3_set_auxdata(). +D 2011-06-14T14:18:45.331 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -237,7 +237,7 @@ F src/update.c 80d77311d91ebc06b27149e75701f1b3e9356622 F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c edfa3827d7a6fac2425bc10c0eb6e54342d2fa56 +F src/vdbe.c 9eef9bb0d4a0de06cf904476eadf58395971e35a F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1 F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 F src/vdbeapi.c 0eeadc75e44a30efd996d6af6e7c5a2488e35be8 @@ -555,7 +555,7 @@ F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 9d7bbfcc1b52c88ba7b2ba6554068ecf9939f252 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test e56c9c3358da2c18385aea15a42dc970913986c2 +F test/malloc.test 76017be66cec4375a4b4ea5c71245e27a9fe2d0b F test/malloc3.test 4128b1e6ffa506103b278ad97af89174f310c7ca F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test 4d16d1bb26d2deddd7c4f480deec341f9b2d0e22 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 77f01578bb565d1bc884b374b68bae10ce34a084 aefd46dfae7e06fbaf4f2b9a86a7f2ac6927331e -R 84da75d7a3a06308ee488ec7371657d8 +P b1f9c1e0ac51cedfb05ac073a603343f6df865b5 +R 915dabb427b7d44a9631a4d5ac3ba884 U dan -Z 26ae4ebe97429ca9da8c05064a989555 +Z 905532115e84db070a1615e7e8131374 diff --git a/manifest.uuid b/manifest.uuid index 0816af7d24..3952312e3e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b1f9c1e0ac51cedfb05ac073a603343f6df865b5 \ No newline at end of file +0185c4b689d18d66e6aa39b4a7bddc279e3c9d17 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 97a0a6a555..a55cee1e99 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1399,15 +1399,6 @@ case OP_Function: { db->lastRowid = lastRowid; (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ lastRowid = db->lastRowid; - if( db->mallocFailed ){ - /* Even though a malloc() has failed, the implementation of the - ** user function may have called an sqlite3_result_XXX() function - ** to return a value. The following call releases any resources - ** associated with such a value. - */ - sqlite3VdbeMemRelease(&ctx.s); - goto no_mem; - } /* If any auxiliary data functions have been called by this user function, ** immediately call the destructor for any non-static values. @@ -1418,6 +1409,16 @@ case OP_Function: { pOp->p4type = P4_VDBEFUNC; } + if( db->mallocFailed ){ + /* Even though a malloc() has failed, the implementation of the + ** user function may have called an sqlite3_result_XXX() function + ** to return a value. The following call releases any resources + ** associated with such a value. + */ + sqlite3VdbeMemRelease(&ctx.s); + goto no_mem; + } + /* If the function returned an error, throw an exception */ if( ctx.isError ){ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); diff --git a/test/malloc.test b/test/malloc.test index 678b2be075..9bd5314bf1 100644 --- a/test/malloc.test +++ b/test/malloc.test @@ -894,6 +894,18 @@ ifcapable stat2&&utf16 { } } +# Test that if an OOM error occurs, aux-data is still correctly destroyed. +# This test case was causing either a memory-leak or an assert() failure +# at one point, depending on the configuration. +# +do_malloc_test 39 -tclprep { + sqlite3 db test.db +} -sqlbody { + SELECT test_auxdata('abc', 'def'); +} -cleanup { + db close +} + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} From 376d1e5c4fc3ebedee9e2323e452161581e1087e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Jun 2011 08:30:29 +0000 Subject: [PATCH 52/69] Add an interface to better test incremental loading of doclists by FTS4. Also some tests for this and term prefix queries. FossilOrigin-Name: 7a3813138d1a5c5d99f4756c79998831d779774f --- ext/fts3/fts3_test.c | 74 +++++++++++++++++++++++++++++++++++++++++-- ext/fts3/fts3_write.c | 18 +++++++---- manifest | 16 +++++----- manifest.uuid | 2 +- test/fts3auto.test | 47 +++++++++++++++++++++++++++ 5 files changed, 140 insertions(+), 17 deletions(-) diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index e82fca608a..de20e06cce 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -242,8 +242,78 @@ static int fts3_near_match_cmd( return rc; } -int Sqlitetestfts3_Init(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); +/* +** Tclcmd: fts3_configure_incr_load ?CHUNKSIZE THRESHOLD? +** +** Normally, FTS uses hard-coded values to determine the minimum doclist +** size eligible for incremental loading, and the size of the chunks loaded +** when a doclist is incrementally loaded. This command allows the built-in +** values to be overridden for testing purposes. +** +** If present, the first argument is the chunksize in bytes to load doclists +** in. The second argument is the minimum doclist size in bytes to use +** incremental loading with. +** +** Whether or not the arguments are present, this command returns a list of +** two integers - the initial chunksize and threshold when the command is +** invoked. This can be used to restore the default behaviour after running +** tests. For example: +** +** # Override incr-load settings for testing: +** set cfg [fts3_configure_incr_load $new_chunksize $new_threshold] +** +** .... run tests .... +** +** # Restore initial incr-load settings: +** eval fts3_configure_incr_load $cfg +*/ +static int fts3_configure_incr_load_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + extern int test_fts3_node_chunksize; + extern int test_fts3_node_chunk_threshold; + int iArg1; + int iArg2; + Tcl_Obj *pRet; + + if( objc!=1 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?CHUNKSIZE THRESHOLD?"); + return TCL_ERROR; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + Tcl_ListObjAppendElement( + interp, pRet, Tcl_NewIntObj(test_fts3_node_chunksize)); + Tcl_ListObjAppendElement( + interp, pRet, Tcl_NewIntObj(test_fts3_node_chunk_threshold)); + + if( objc==3 ){ + int iArg1; + int iArg2; + if( Tcl_GetIntFromObj(interp, objv[1], &iArg1) + || Tcl_GetIntFromObj(interp, objv[2], &iArg2) + ){ + Tcl_DecrRefCount(pRet); + return TCL_ERROR; + } + test_fts3_node_chunksize = iArg1; + test_fts3_node_chunk_threshold = iArg2; + } + + Tcl_SetObjResult(interp, pRet); + Tcl_DecrRefCount(pRet); + return TCL_OK; +} + +int Sqlitetestfts3_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); + Tcl_CreateObjCommand(interp, + "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0 + ); return TCL_OK; } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 30c0a1a15b..0d1e6ba21d 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -49,13 +49,19 @@ ** is 1. Clearly such small values would be inefficient, but can be useful ** for testing purposes. ** -** TODO: Add a test interface to modify these "constants" from a script for -** this purpose. +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. */ -#define FTS3_NODE_CHUNKSIZE (4*1024) -#define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) -/* #define FTS3_NODE_CHUNKSIZE 1 */ -/* #define FTS3_NODE_CHUNK_THRESHOLD 1 */ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; diff --git a/manifest b/manifest index 54f991e432..5c19d15b5f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sthat\scan\sfollow\san\sOOM\serror\sin\sa\suser-function\sthat\suses\ssqlite3_set_auxdata(). -D 2011-06-14T14:18:45.331 +C Add\san\sinterface\sto\sbetter\stest\sincremental\sloading\sof\sdoclists\sby\sFTS4.\sAlso\ssome\stests\sfor\sthis\sand\sterm\sprefix\squeries. +D 2011-06-15T08:30:29.176 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -72,11 +72,11 @@ F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 F ext/fts3/fts3_snippet.c 82e2c1e420c871c02f6e85ea438570118d7105c8 F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 -F ext/fts3/fts3_test.c 9376cc865447e63c671f0f9ffd1a2c9a29678230 +F ext/fts3/fts3_test.c 4c90a63c117c989ea899d59e2195622dcf61dbc0 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c bc24cec303d86aeb4b40fcbdf9f252f93ef78fc7 +F ext/fts3/fts3_write.c 4c008450666d6a1f45ba404cf654ebdb3d4da4cd F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test 2f86f2a0e8ffa26d81d570897e6cc1c2262256d5 +F test/fts3auto.test a98cc895bd92df14ce4a6e94f5c68d33edcc1372 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P b1f9c1e0ac51cedfb05ac073a603343f6df865b5 -R 915dabb427b7d44a9631a4d5ac3ba884 +P 0185c4b689d18d66e6aa39b4a7bddc279e3c9d17 +R 1c51f9714aac7706b1e74e389e8d32c5 U dan -Z 905532115e84db070a1615e7e8131374 +Z 77f3043182a3d5e0d927f56eefb2694c diff --git a/manifest.uuid b/manifest.uuid index 3952312e3e..34ba6c7297 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0185c4b689d18d66e6aa39b4a7bddc279e3c9d17 \ No newline at end of file +7a3813138d1a5c5d99f4756c79998831d779774f \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 5c9b85128b..1fd02c537f 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -530,6 +530,53 @@ foreach {tn create} { do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*} } +#-------------------------------------------------------------------------- +# The following test cases - fts3auto-5.* - focus on using prefix indexes. +# +set chunkconfig [fts3_configure_incr_load 1 1] +foreach {tn create} { + 1 "fts4(a, b)" + 2 "fts4(a, b, order=DESC, prefix=1)" + 3 "fts4(a, b, order=ASC, prefix=1,3)" + 4 "fts4(a, b, order=DESC, prefix=2,4)" +} { + + execsql [subst { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING $create; + }] + + foreach {a b} { + "the song of songs which is solomons" + "let him kiss me with the kisses of his mouth for thy love is better than wine" + "because of the savour of thy good ointments thy name is as ointment poured forth therefore do the virgins love thee" + "draw me we will run after thee the king hath brought me into his chambers we will be glad and rejoice in thee we will remember thy love more than wine the upright love thee" + "i am black but comely o ye daughters of jerusalem as the tents of kedar as the curtains of solomon" + "look not upon me because i am black because the sun hath looked upon me my mothers children were angry with me they made me the keeper of the vineyards but mine own vineyard have i not kept" + "tell me o thou whom my soul loveth where thou feedest where thou makest thy flock to rest at noon for why should i be as one that turneth aside by the flocks of thy companions?" + "if thou know not o thou fairest among women go thy way forth by the footsteps of the flock and feed thy kids beside the shepherds tents" + "i have compared thee o my love to a company of horses in pharaohs chariots" + "thy cheeks are comely with rows of jewels thy neck with chains of gold" + "we will make thee borders of gold with studs of silver" + "while the king sitteth at his table my spikenard sendeth forth the smell thereof" + "a bundle of myrrh is my wellbeloved unto me he shall lie all night betwixt my breasts" + "my beloved is unto me as a cluster of camphire in the vineyards of en gedi" + "behold thou art fair my love behold thou art fair thou hast doves eyes" + "behold thou art fair my beloved yea pleasant also our bed is green" + "the beams of our house are cedar and our rafters of fir" + } { + execsql {INSERT INTO t1(a, b) VALUES($a, $b)} + } + + do_fts3query_test 5.$tn.1.1 t1 {s*} + do_fts3query_test 5.$tn.1.2 t1 {so*} + do_fts3query_test 5.$tn.1.3 t1 {"s* o*"} + do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*} + do_fts3query_test 5.$tn.1.5 t1 {th* NEAR/5 a* NEAR/5 w*} + do_fts3query_test 5.$tn.1.6 t1 {"b* th* art* fair*"} +} +eval fts3_configure_incr_load $chunkconfig + set sqlite_fts3_enable_parentheses $sfep finish_test From 1aa4f3e52924c30a9ddcbd18adc672c72a25fe24 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Jun 2011 12:43:36 +0000 Subject: [PATCH 53/69] Fix a couple of compiler warnings. FossilOrigin-Name: 3899f3b95ed50773a72d907b39b451fefce69c9e --- ext/fts3/fts3.c | 3 +-- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/func.c | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index feed5220c9..988a1a65c6 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3275,12 +3275,11 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nPoslist = nList; }else{ - assert( iPrev>=0 ); - char *aOut = pList; char *p1 = aPoslist; char *p2 = aOut; + assert( iPrev>=0 ); fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); sqlite3_free(aPoslist); aPoslist = pList; diff --git a/manifest b/manifest index 5c19d15b5f..a53e232c7e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sinterface\sto\sbetter\stest\sincremental\sloading\sof\sdoclists\sby\sFTS4.\sAlso\ssome\stests\sfor\sthis\sand\sterm\sprefix\squeries. -D 2011-06-15T08:30:29.176 +C Fix\sa\scouple\sof\scompiler\swarnings. +D 2011-06-15T12:43:36.849 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c ae3ae91d204d177fc37a22c7bc5a3a9df70e65a0 +F ext/fts3/fts3.c 6c7b588761082e3c5452fbff02c8a2004875ea26 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -135,7 +135,7 @@ F src/delete.c cecc926c70783452f3e8eb452c728291ce1a0b21 F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 9fabba17a4d4778dc660f0cb9d781fc86d7b9d41 -F src/func.c d93772d9ffa51e4a8f275675bae1f61394c4ab80 +F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7 F src/global.c 29bfb85611dd816b04f10fba0ca910366e128d38 F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 0185c4b689d18d66e6aa39b4a7bddc279e3c9d17 -R 1c51f9714aac7706b1e74e389e8d32c5 -U dan -Z 77f3043182a3d5e0d927f56eefb2694c +P 7a3813138d1a5c5d99f4756c79998831d779774f +R e8d9fd64770806090874e1cc045d5b07 +U drh +Z ea6a718ef2a7cf11b3bc2eae3fdf9321 diff --git a/manifest.uuid b/manifest.uuid index 34ba6c7297..a17cc513fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a3813138d1a5c5d99f4756c79998831d779774f \ No newline at end of file +3899f3b95ed50773a72d907b39b451fefce69c9e \ No newline at end of file diff --git a/src/func.c b/src/func.c index 407ea26aa6..16de6bbbdf 100644 --- a/src/func.c +++ b/src/func.c @@ -608,7 +608,7 @@ static int patternCompare( return 0; } }else if( c==matchSet ){ - int prior_c = 0; + u32 prior_c = 0; assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ seen = 0; invert = 0; From bf4bc8c25483984244c37485ff9b689ab1996faa Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Jun 2011 13:02:21 +0000 Subject: [PATCH 54/69] Fix so that the TCL test harness works even if SQLITE_ENABLE_FTS is omitted. FossilOrigin-Name: 63ebcb52a1909aca80f2fef3e982f8fb5929b73b --- ext/fts3/fts3_test.c | 3 ++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index de20e06cce..704c7502af 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -273,6 +273,7 @@ static int fts3_configure_incr_load_cmd( int objc, Tcl_Obj *CONST objv[] ){ +#ifdef SQLITE_ENABLE_FTS3 extern int test_fts3_node_chunksize; extern int test_fts3_node_chunk_threshold; int iArg1; @@ -306,6 +307,7 @@ static int fts3_configure_incr_load_cmd( Tcl_SetObjResult(interp, pRet); Tcl_DecrRefCount(pRet); +#endif return TCL_OK; } @@ -316,4 +318,3 @@ int Sqlitetestfts3_Init(Tcl_Interp *interp){ ); return TCL_OK; } - diff --git a/manifest b/manifest index a53e232c7e..cbdd4b17de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\scompiler\swarnings. -D 2011-06-15T12:43:36.849 +C Fix\sso\sthat\sthe\sTCL\stest\sharness\sworks\seven\sif\sSQLITE_ENABLE_FTS\sis\somitted. +D 2011-06-15T13:02:21.324 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -72,7 +72,7 @@ F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 F ext/fts3/fts3_snippet.c 82e2c1e420c871c02f6e85ea438570118d7105c8 F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 -F ext/fts3/fts3_test.c 4c90a63c117c989ea899d59e2195622dcf61dbc0 +F ext/fts3/fts3_test.c 4e833729c13cea9a6bb98d3b353f6e3b8f756004 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 7a3813138d1a5c5d99f4756c79998831d779774f -R e8d9fd64770806090874e1cc045d5b07 +P 3899f3b95ed50773a72d907b39b451fefce69c9e +R fc57a2e2686d2dead5f714ec2e85c3f0 U drh -Z ea6a718ef2a7cf11b3bc2eae3fdf9321 +Z 950ecff3e7df79fb4044b7edf191f8e4 diff --git a/manifest.uuid b/manifest.uuid index a17cc513fc..e8c5b891b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3899f3b95ed50773a72d907b39b451fefce69c9e \ No newline at end of file +63ebcb52a1909aca80f2fef3e982f8fb5929b73b \ No newline at end of file From 7c65a882f3af98f3dd9cf13539a20bc48626c3d3 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Jun 2011 16:07:30 +0000 Subject: [PATCH 55/69] Fix the wal7.test script so that it works even if secure_delete is engaged. FossilOrigin-Name: 68fb7a548c8fe00bcb9c6d71f2863b55d42b3c64 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal7.test | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 69164f4cd0..1b4b756d23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\simproved\sincremental\sdoclist\sloading\stest\sfrom\sthe\nbroken-build\sbranch\s(which\sis\snow\sfixed)\sinto\strunk. -D 2011-06-15T13:11:06.128 +C Fix\sthe\swal7.test\sscript\sso\sthat\sit\sworks\seven\sif\ssecure_delete\sis\sengaged. +D 2011-06-15T16:07:30.722 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -878,7 +878,7 @@ F test/wal3.test 5c396cc22497244d627306f4c1d360167353f8dd F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 F test/wal5.test 1bbfaa316dc2a1d0d1fac3f4500c38a90055a41b F test/wal6.test 07aa31ca8892d0527f2c5c5a9a2a87aa421dfaa8 -F test/wal7.test 09bc8de3d11949571d6f7a4188b308059cec27e5 +F test/wal7.test c9c8d43bbfc7b99e49501b25c06506700e044891 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 0185c4b689d18d66e6aa39b4a7bddc279e3c9d17 63ebcb52a1909aca80f2fef3e982f8fb5929b73b -R fc57a2e2686d2dead5f714ec2e85c3f0 +P f9750870ee04935f338e4d808900fee5a8b2b389 +R 1072d6428994e2a6ec2fd048ac7a1c15 U drh -Z ece8b2c59ad0a86fa0f8629e77710c29 +Z ba8c39e1b220c272ee066a1f42d23d01 diff --git a/manifest.uuid b/manifest.uuid index 2089caf185..8ca0413a2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9750870ee04935f338e4d808900fee5a8b2b389 \ No newline at end of file +68fb7a548c8fe00bcb9c6d71f2863b55d42b3c64 \ No newline at end of file diff --git a/test/wal7.test b/test/wal7.test index cfe2d7b8b0..ee14f0fb77 100644 --- a/test/wal7.test +++ b/test/wal7.test @@ -61,7 +61,7 @@ do_test wal7-2.0 { INSERT INTO t1 VALUES(zeroblob(200000),4); CREATE TABLE t2(z); DELETE FROM t1; - INSERT INTO t2 SELECT x FROM t1; + INSERT INTO t2 VALUES(1); } file size test.db-wal } 25000 @@ -83,7 +83,7 @@ do_test wal7-3.0 { INSERT INTO t1 VALUES(zeroblob(200000),4); CREATE TABLE t2(z); DELETE FROM t1; - INSERT INTO t2 SELECT x FROM t1; + INSERT INTO t2 VALUES(1); } set sz [file size test.db-wal] expr {$sz>0 && $sz<10000} @@ -106,7 +106,7 @@ do_test wal7-4.0 { INSERT INTO t1 VALUES(zeroblob(200000),4); CREATE TABLE t2(z); DELETE FROM t1; - INSERT INTO t2 SELECT x FROM t1; + INSERT INTO t2 VALUES(1); } set sz [file size test.db-wal] } 25000 From b0a328324f8114a1da8c38394a70796d51222c78 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Jun 2011 17:04:43 +0000 Subject: [PATCH 56/69] Add a couple of pointer type casts to test file test_quota.c. FossilOrigin-Name: 0df061b0554c749cade4ec8ddabe453934825bb2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_quota.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1b4b756d23..b225447438 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\swal7.test\sscript\sso\sthat\sit\sworks\seven\sif\ssecure_delete\sis\sengaged. -D 2011-06-15T16:07:30.722 +C Add\sa\scouple\sof\spointer\stype\scasts\sto\stest\sfile\stest_quota.c. +D 2011-06-15T17:04:43.219 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec F src/test_osinst.c 62b0b8ef21ce754cc94e17bb42377ed8795dba32 F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8 -F src/test_quota.c b5576f17d701af461effd7ca1e71f0d100071192 +F src/test_quota.c cc4f67e12558a252ea4a11720be268348f4b1595 F src/test_rtree.c 30c981837445a4e187ee850a49c4760d9642f7c3 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P f9750870ee04935f338e4d808900fee5a8b2b389 -R 1072d6428994e2a6ec2fd048ac7a1c15 -U drh -Z ba8c39e1b220c272ee066a1f42d23d01 +P 68fb7a548c8fe00bcb9c6d71f2863b55d42b3c64 +R c0385c1daebb63351a38fc46b33ad8a2 +U dan +Z 77d8ea6289891695c9c538ba7b8f6716 diff --git a/manifest.uuid b/manifest.uuid index 8ca0413a2d..4c6513b5b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68fb7a548c8fe00bcb9c6d71f2863b55d42b3c64 \ No newline at end of file +0df061b0554c749cade4ec8ddabe453934825bb2 \ No newline at end of file diff --git a/src/test_quota.c b/src/test_quota.c index 3c6db4d7cd..9b0e4a9ddc 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -323,7 +323,7 @@ static int quotaOpen( pFile=pFile->pNext){} if( pFile==0 ){ int nName = strlen(zName); - pFile = sqlite3_malloc( sizeof(*pFile) + nName + 1 ); + pFile = (quotaFile *)sqlite3_malloc( sizeof(*pFile) + nName + 1 ); if( pFile==0 ){ quotaLeave(); pSubOpen->pMethods->xClose(pSubOpen); @@ -683,7 +683,7 @@ int sqlite3_quota_set( quotaLeave(); return SQLITE_OK; } - pGroup = sqlite3_malloc( sizeof(*pGroup) + nPattern + 1 ); + pGroup = (quotaGroup *)sqlite3_malloc( sizeof(*pGroup) + nPattern + 1 ); if( pGroup==0 ){ quotaLeave(); return SQLITE_NOMEM; From 518d656574aa4e10a2e4d472f36fae547917459d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Jun 2011 19:18:47 +0000 Subject: [PATCH 57/69] Further tweaks to the wal7.test test case. FossilOrigin-Name: 177e2d72a8a654d25720e6afc7706624610ac80f --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/wal7.test | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b225447438..7f46530224 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scouple\sof\spointer\stype\scasts\sto\stest\sfile\stest_quota.c. -D 2011-06-15T17:04:43.219 +C Further\stweaks\sto\sthe\swal7.test\stest\scase. +D 2011-06-15T19:18:47.862 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -878,7 +878,7 @@ F test/wal3.test 5c396cc22497244d627306f4c1d360167353f8dd F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 F test/wal5.test 1bbfaa316dc2a1d0d1fac3f4500c38a90055a41b F test/wal6.test 07aa31ca8892d0527f2c5c5a9a2a87aa421dfaa8 -F test/wal7.test c9c8d43bbfc7b99e49501b25c06506700e044891 +F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 68fb7a548c8fe00bcb9c6d71f2863b55d42b3c64 -R c0385c1daebb63351a38fc46b33ad8a2 -U dan -Z 77d8ea6289891695c9c538ba7b8f6716 +P 0df061b0554c749cade4ec8ddabe453934825bb2 +R b35795e0bbb3ce3458e3327edc3421f8 +U drh +Z 625133d76b14b6cbfe340f5d4f1b609e diff --git a/manifest.uuid b/manifest.uuid index 4c6513b5b7..c41c28f792 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0df061b0554c749cade4ec8ddabe453934825bb2 \ No newline at end of file +177e2d72a8a654d25720e6afc7706624610ac80f \ No newline at end of file diff --git a/test/wal7.test b/test/wal7.test index ee14f0fb77..cacfaedacb 100644 --- a/test/wal7.test +++ b/test/wal7.test @@ -86,7 +86,7 @@ do_test wal7-3.0 { INSERT INTO t2 VALUES(1); } set sz [file size test.db-wal] - expr {$sz>0 && $sz<10000} + expr {$sz>0 && $sz<13700} } 1 From babb61f3465789cc0eec99455217fcd97084bd57 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Jun 2011 23:34:51 +0000 Subject: [PATCH 58/69] Fix the multiplex.test module so that it works with the inmemory_journal permutation. FossilOrigin-Name: 03d9480fc4a8310bc8da81a64f9206c8f4b501d9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/multiplex.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7f46530224..fa4aff42a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stweaks\sto\sthe\swal7.test\stest\scase. -D 2011-06-15T19:18:47.862 +C Fix\sthe\smultiplex.test\smodule\sso\sthat\sit\sworks\swith\sthe\sinmemory_journal\npermutation. +D 2011-06-15T23:34:51.929 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -593,7 +593,7 @@ F test/misc5.test 45b2e3ed5f79af2b4f38ae362eaf4c49674575bd F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test 29032efcd3d826fbd409e2a7af873e7939f4a4e3 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 -F test/multiplex.test 7a8a50c8ed72dfcf4db9ebae977f7a63184639d8 +F test/multiplex.test 555080c87abfc72ba68e2f3df01d4a9a7a4fdf58 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test dc212a22b36109fd1ae37154292444ef249c5ec2 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 0df061b0554c749cade4ec8ddabe453934825bb2 -R b35795e0bbb3ce3458e3327edc3421f8 +P 177e2d72a8a654d25720e6afc7706624610ac80f +R e264f27c78f465a3a7dd939a16ceb912 U drh -Z 625133d76b14b6cbfe340f5d4f1b609e +Z 199fa6f213cb3bfff11b26a0ef6f9d7c diff --git a/manifest.uuid b/manifest.uuid index c41c28f792..1618bb323a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -177e2d72a8a654d25720e6afc7706624610ac80f \ No newline at end of file +03d9480fc4a8310bc8da81a64f9206c8f4b501d9 \ No newline at end of file diff --git a/test/multiplex.test b/test/multiplex.test index ae60d639e5..9278e84274 100644 --- a/test/multiplex.test +++ b/test/multiplex.test @@ -572,6 +572,7 @@ if {0==[info exists ::G(perm:presql)] || $::G(perm:presql) == ""} { ifcapable vacuum { +sqlite3_multiplex_shutdown do_test multiplex-6.0.0 { multiplex_delete test.db sqlite3_multiplex_initialize "" 1 From 6b96771b821bdff9db39bd51b5923d846c92bc69 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Jun 2011 00:54:45 +0000 Subject: [PATCH 59/69] Changes to #ifdefs so that the build goes correctly if the only FTS macro defined is SQLITE_ENABLE_FTS4. FossilOrigin-Name: a0b43a320e6491de7070966ed7c3ec55fd660a85 --- ext/fts3/fts3.c | 3 +-- ext/fts3/fts3Int.h | 13 +++++++++++-- ext/fts3/fts3_aux.c | 3 +-- ext/fts3/fts3_expr.c | 2 +- ext/fts3/fts3_hash.c | 1 + ext/fts3/fts3_icu.c | 4 +--- ext/fts3/fts3_porter.c | 3 +-- ext/fts3/fts3_snippet.c | 2 +- ext/fts3/fts3_term.c | 2 +- ext/fts3/fts3_tokenizer.c | 6 +++--- ext/fts3/fts3_tokenizer1.c | 3 +-- ext/fts3/fts3_write.c | 2 +- manifest | 34 +++++++++++++++++----------------- manifest.uuid | 2 +- 14 files changed, 42 insertions(+), 38 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 988a1a65c6..024b1150a2 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -292,14 +292,13 @@ ** into a single segment. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) # define SQLITE_CORE 1 #endif -#include "fts3Int.h" - #include #include #include diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 80604d6fa3..06f2c09bb8 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -11,7 +11,6 @@ ****************************************************************************** ** */ - #ifndef _FTSINT_H #define _FTSINT_H @@ -19,6 +18,16 @@ # define NDEBUG 1 #endif +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +#ifdef SQLITE_ENABLE_FTS3 #include "sqlite3.h" #include "fts3_tokenizer.h" #include "fts3_hash.h" @@ -502,5 +511,5 @@ int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); - +#endif /* SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 160eff981a..ada85d796b 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -11,10 +11,9 @@ ****************************************************************************** ** */ - +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#include "fts3Int.h" #include #include diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 0383d1a276..7eb2962d44 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -15,6 +15,7 @@ ** syntax is relatively simple, the whole tokenizer/parser system is ** hand-coded. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* @@ -77,7 +78,6 @@ int sqlite3_fts3_enable_parentheses = 0; */ #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 -#include "fts3Int.h" #include #include diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c index 98be529605..b7c3e8b5a7 100644 --- a/ext/fts3/fts3_hash.c +++ b/ext/fts3/fts3_hash.c @@ -23,6 +23,7 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include diff --git a/ext/fts3/fts3_icu.c b/ext/fts3/fts3_icu.c index 85390d3b06..a10a55d67b 100644 --- a/ext/fts3/fts3_icu.c +++ b/ext/fts3/fts3_icu.c @@ -10,10 +10,8 @@ ** ************************************************************************* ** This file implements a tokenizer for fts3 based on the ICU library. -** -** $Id: fts3_icu.c,v 1.3 2008/09/01 18:34:20 danielk1977 Exp $ */ - +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_ENABLE_ICU diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index 27f9cf39d7..148c570088 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -22,9 +22,8 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include #include diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 5ae3a16fc4..0da370c7ea 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -11,9 +11,9 @@ ****************************************************************************** */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#include "fts3Int.h" #include #include diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index eff65b7bd1..da9bf4543b 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -15,10 +15,10 @@ ** access to the full-text index of an FTS table. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_TEST -#include "fts3Int.h" #include #include diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index aa28933439..87a9a3d430 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -23,14 +23,14 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #include "sqlite3ext.h" #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #endif - #include "fts3Int.h" + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + #include #include diff --git a/ext/fts3/fts3_tokenizer1.c b/ext/fts3/fts3_tokenizer1.c index 432c35d1a2..d11a499766 100644 --- a/ext/fts3/fts3_tokenizer1.c +++ b/ext/fts3/fts3_tokenizer1.c @@ -22,9 +22,8 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include #include diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 0d1e6ba21d..ea59e6eb62 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -17,9 +17,9 @@ ** code in fts3.c. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#include "fts3Int.h" #include #include #include diff --git a/manifest b/manifest index fa4aff42a9..d847584939 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\smultiplex.test\smodule\sso\sthat\sit\sworks\swith\sthe\sinmemory_journal\npermutation. -D 2011-06-15T23:34:51.929 +C Changes\sto\s#ifdefs\sso\sthat\sthe\sbuild\sgoes\scorrectly\sif\sthe\sonly\sFTS\smacro\ndefined\sis\sSQLITE_ENABLE_FTS4. +D 2011-06-16T00:54:45.816 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,22 +61,22 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 6c7b588761082e3c5452fbff02c8a2004875ea26 +F ext/fts3/fts3.c 014e06f585f7ff0e34229a8eb7955ae03b8086fe F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 -F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe -F ext/fts3/fts3_expr.c b95f0d76bcf4507c73a838f3178c4ed8c42dc2bb -F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c +F ext/fts3/fts3Int.h 27b0b75c4d50f4022eeacbd62801d8e204ae2584 +F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 +F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b +F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec -F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 -F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c 82e2c1e420c871c02f6e85ea438570118d7105c8 -F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 +F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa +F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c +F ext/fts3/fts3_snippet.c a44b38a07d39701ab6d20d7d89fcafe193bf3680 +F ext/fts3/fts3_term.c 51e384269edcc015e8b555fdad2338f053388975 F ext/fts3/fts3_test.c 4e833729c13cea9a6bb98d3b353f6e3b8f756004 -F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d +F ext/fts3/fts3_tokenizer.c 90ba6cdd8bb1b3686ab7a3d72333131e13c8fdb2 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 -F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c 4c008450666d6a1f45ba404cf654ebdb3d4da4cd +F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 +F ext/fts3/fts3_write.c b0df39416b69e735c1209a2c1cd61bc3cd21c2c9 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 177e2d72a8a654d25720e6afc7706624610ac80f -R e264f27c78f465a3a7dd939a16ceb912 +P 03d9480fc4a8310bc8da81a64f9206c8f4b501d9 +R d0766e6b55a873753b193b31dc04f4d0 U drh -Z 199fa6f213cb3bfff11b26a0ef6f9d7c +Z d9decbc0f2b53d6559c9602b3b81dc73 diff --git a/manifest.uuid b/manifest.uuid index 1618bb323a..19f7afc0ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03d9480fc4a8310bc8da81a64f9206c8f4b501d9 \ No newline at end of file +a0b43a320e6491de7070966ed7c3ec55fd660a85 \ No newline at end of file From d4d21fea8e6f9e8ad262a42886a581257f5e2402 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Jun 2011 16:06:05 +0000 Subject: [PATCH 60/69] Fix a problem with NEAR queries executed inside a transaction that writes the FTS table. FossilOrigin-Name: 051c756c367837908f6691c0a36108e088c94f99 --- ext/fts3/fts3.c | 15 ++-- ext/fts3/fts3Int.h | 1 + ext/fts3/fts3_write.c | 200 ++++++++++++++++++++++++++++-------------- manifest | 20 ++--- manifest.uuid | 2 +- test/fts3auto.test | 26 ++++-- 6 files changed, 173 insertions(+), 91 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 024b1150a2..f2c907c8c0 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3493,7 +3493,7 @@ static int fts3EvalPhraseNext( while( pIterpNextDocid = pIter; - assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] ); + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); *pbEof = 0; } } @@ -4131,13 +4131,14 @@ static void fts3EvalRestart( if( pPhrase ){ fts3EvalZeroPoslist(pPhrase); if( pPhrase->bIncr ){ - sqlite3Fts3EvalPhraseCleanup(pPhrase); - memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); - *pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0); - }else{ - pPhrase->doclist.pNextDocid = 0; - pPhrase->doclist.iDocid = 0; + assert( pPhrase->nToken==1 ); + assert( pPhrase->aToken[0].pSegcsr ); + sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); } + + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; } pExpr->iDocid = 0; diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 06f2c09bb8..5d3d22a57a 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -439,6 +439,7 @@ struct Fts3MultiSegReader { int nBuffer; /* Allocated size of aBuffer[] in bytes */ int iColFilter; /* If >=0, filter for this column */ + int bRestart; /* Used by fts3.c only. */ int nCost; /* Cost of running iterator */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index ea59e6eb62..da84d6ca61 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1247,6 +1247,7 @@ static int fts3SegReaderNextDocid( pReader->pOffsetList = p; } }else{ + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; /* Pointer p currently points at the first byte of an offset list. The ** following block advances it to point one byte past the end of @@ -1275,13 +1276,15 @@ static int fts3SegReaderNextDocid( *ppOffsetList = pReader->pOffsetList; *pnOffsetList = (int)(p - pReader->pOffsetList - 1); } + + while( p=&pReader->aDoclist[pReader->nDoclist] ){ + if( p>=pEnd ){ pReader->pOffsetList = 0; }else{ rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); @@ -2265,51 +2268,27 @@ static void fts3ColumnFilter( *pnList = nList; } -int sqlite3Fts3MsrIncrStart( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pCsr, /* Cursor object */ - int iCol, /* Column to match on. */ - const char *zTerm, /* Term to iterate through a doclist for */ - int nTerm /* Number of bytes in zTerm */ +/* +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any +** existing data). Grow the buffer if required. +** +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered +** trying to resize the buffer, return SQLITE_NOMEM. +*/ +static int fts3MsrBufferData( + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + char *pList, + int nList ){ - int i; - int nSegment = pCsr->nSegment; - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( - p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp - ); - - assert( pCsr->pFilter==0 ); - assert( zTerm && nTerm>0 ); - - /* Advance each segment iterator until it points to the term zTerm/nTerm. */ - for(i=0; iapSegment[i]; - do { - int rc = fts3SegReaderNext(p, pSeg, 1); - if( rc!=SQLITE_OK ) return rc; - }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); + if( nList>pMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; } - fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp); - - /* Determine how many of the segments actually point to zTerm/nTerm. */ - for(i=0; iapSegment[i]; - if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ - break; - } - } - pCsr->nAdvance = i; - - /* Advance each of the segments to point to the first docid. */ - for(i=0; inAdvance; i++){ - int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); - if( rc!=SQLITE_OK ) return rc; - } - fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); - - assert( iCol<0 || iColnColumn ); - pCsr->iColFilter = iCol; + memcpy(pMsr->aBuffer, pList, nList); return SQLITE_OK; } @@ -2363,49 +2342,135 @@ int sqlite3Fts3MsrIncrNext( } if( nList>0 ){ + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + *paPoslist = pMsr->aBuffer; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + }else{ + *paPoslist = pList; + } *piDocid = iDocid; - *paPoslist = pList; *pnPoslist = nList; break; } } - } return SQLITE_OK; } +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ +){ + int i; + int nSeg = pCsr->nSegment; + + /* If the Fts3SegFilter defines a specific term (or term prefix) to search + ** for, then advance each segment iterator until it points to a term of + ** equal or greater value than the specified term. This prevents many + ** unnecessary merge/sort operations for the case where single segment + ** b-tree leaf nodes contain more than one term. + */ + for(i=0; pCsr->bRestart==0 && inSegment; i++){ + Fts3SegReader *pSeg = pCsr->apSegment[i]; + do { + int rc = fts3SegReaderNext(p, pSeg, 0); + if( rc!=SQLITE_OK ) return rc; + }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); + } + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); + + return SQLITE_OK; +} + int sqlite3Fts3SegReaderStart( Fts3Table *p, /* Virtual table handle */ Fts3MultiSegReader *pCsr, /* Cursor object */ Fts3SegFilter *pFilter /* Restrictions on range of iteration */ ){ - int i; - - /* Initialize the cursor object */ pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} - /* If the Fts3SegFilter defines a specific term (or term prefix) to search - ** for, then advance each segment iterator until it points to a term of - ** equal or greater value than the specified term. This prevents many - ** unnecessary merge/sort operations for the case where single segment - ** b-tree leaf nodes contain more than one term. - */ - for(i=0; inSegment; i++){ - int nTerm = pFilter->nTerm; - const char *zTerm = pFilter->zTerm; +int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; - do { - int rc = fts3SegReaderNext(p, pSeg, 0); - if( rc!=SQLITE_OK ) return rc; - }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } } - fts3SegReaderSort( - pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; return SQLITE_OK; } +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + + int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ Fts3MultiSegReader *pCsr /* Cursor object */ @@ -2478,9 +2543,14 @@ int sqlite3Fts3SegReaderStep( && !isIgnoreEmpty && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) ){ - pCsr->aDoclist = apSegment[0]->aDoclist; pCsr->nDoclist = apSegment[0]->nDoclist; - rc = SQLITE_ROW; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ diff --git a/manifest b/manifest index d847584939..2efad53b59 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\s#ifdefs\sso\sthat\sthe\sbuild\sgoes\scorrectly\sif\sthe\sonly\sFTS\smacro\ndefined\sis\sSQLITE_ENABLE_FTS4. -D 2011-06-16T00:54:45.816 +C Fix\sa\sproblem\swith\sNEAR\squeries\sexecuted\sinside\sa\stransaction\sthat\swrites\sthe\sFTS\stable. +D 2011-06-16T16:06:05.320 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,9 +61,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 014e06f585f7ff0e34229a8eb7955ae03b8086fe +F ext/fts3/fts3.c 78b02b5f0195e397c4239ef9213e5506b7d3fa97 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 27b0b75c4d50f4022eeacbd62801d8e204ae2584 +F ext/fts3/fts3Int.h df761492ae2308d3d56123907ca29cdf9bdd3748 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb @@ -76,7 +76,7 @@ F ext/fts3/fts3_test.c 4e833729c13cea9a6bb98d3b353f6e3b8f756004 F ext/fts3/fts3_tokenizer.c 90ba6cdd8bb1b3686ab7a3d72333131e13c8fdb2 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c b0df39416b69e735c1209a2c1cd61bc3cd21c2c9 +F ext/fts3/fts3_write.c 5774a7ee9632355ebf1ec4b7a5071fc9ab9eb956 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test a98cc895bd92df14ce4a6e94f5c68d33edcc1372 +F test/fts3auto.test b0d360b331ff68bd9fb497a6192d23dc0783637c F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 03d9480fc4a8310bc8da81a64f9206c8f4b501d9 -R d0766e6b55a873753b193b31dc04f4d0 -U drh -Z d9decbc0f2b53d6559c9602b3b81dc73 +P a0b43a320e6491de7070966ed7c3ec55fd660a85 +R a91f82245adaecbc51456523e27b5133 +U dan +Z 67955b067165a6636fb02e1c55fe6931 diff --git a/manifest.uuid b/manifest.uuid index 19f7afc0ac..999de7fa63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0b43a320e6491de7070966ed7c3ec55fd660a85 \ No newline at end of file +051c756c367837908f6691c0a36108e088c94f99 \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 1fd02c537f..ed96129a70 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -521,6 +521,7 @@ foreach {tn create} { do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five} do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five} do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five} + do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five} do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*} @@ -534,18 +535,23 @@ foreach {tn create} { # The following test cases - fts3auto-5.* - focus on using prefix indexes. # set chunkconfig [fts3_configure_incr_load 1 1] -foreach {tn create} { - 1 "fts4(a, b)" - 2 "fts4(a, b, order=DESC, prefix=1)" - 3 "fts4(a, b, order=ASC, prefix=1,3)" - 4 "fts4(a, b, order=DESC, prefix=2,4)" +foreach {tn create pending} { + 2 "fts4(a, b, order=ASC, prefix=1)" 1 + + 1 "fts4(a, b)" 1 + 3 "fts4(a, b, order=ASC, prefix=1,3)" 0 + 4 "fts4(a, b, order=DESC, prefix=2,4)" 0 + 5 "fts4(a, b, order=DESC, prefix=1)" 0 + 6 "fts4(a, b, order=ASC, prefix=1,3)" 0 } { execsql [subst { - DROP TABLE t1; + DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING $create; }] + if {$pending} {execsql BEGIN} + foreach {a b} { "the song of songs which is solomons" "let him kiss me with the kisses of his mouth for thy love is better than wine" @@ -568,12 +574,16 @@ foreach {tn create} { execsql {INSERT INTO t1(a, b) VALUES($a, $b)} } + do_fts3query_test 5.$tn.1.1 t1 {s*} do_fts3query_test 5.$tn.1.2 t1 {so*} do_fts3query_test 5.$tn.1.3 t1 {"s* o*"} do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*} - do_fts3query_test 5.$tn.1.5 t1 {th* NEAR/5 a* NEAR/5 w*} - do_fts3query_test 5.$tn.1.6 t1 {"b* th* art* fair*"} + do_fts3query_test 5.$tn.1.5 t1 {a*} + do_fts3query_test 5.$tn.1.6 t1 {th* NEAR/5 a* NEAR/5 w*} + do_fts3query_test 5.$tn.1.7 t1 {"b* th* art* fair*"} + + if {$pending} {execsql COMMIT} } eval fts3_configure_incr_load $chunkconfig From b2f20bfc53b71279a82d15e149bfcf57609ef337 Mon Sep 17 00:00:00 2001 From: shaneh Date: Fri, 17 Jun 2011 07:07:24 +0000 Subject: [PATCH 61/69] Add Microsoft nmake compatible makefile; update a few test cases for Windows. FossilOrigin-Name: a7590af65f3881cf905b8cac2b87381e89a80271 --- Makefile.msc | 865 +++++++++++++++++++++++++++++++++++++++++++ manifest | 17 +- manifest.uuid | 2 +- test/pager1.test | 7 + test/pagerfault.test | 5 + 5 files changed, 887 insertions(+), 9 deletions(-) create mode 100644 Makefile.msc diff --git a/Makefile.msc b/Makefile.msc new file mode 100644 index 0000000000..a65b16dc39 --- /dev/null +++ b/Makefile.msc @@ -0,0 +1,865 @@ +# +# nmake Makefile for SQLITE +# + +# The toplevel directory of the source tree. This is the directory +# that contains this "Makefile.msc". +# +TOP = . + +# C Compiler and options for use in building executables that +# will run on the platform that is doing the build. +# +BCC = cl.exe -O2 + +# C Compile and options for use in building executables that +# will run on the target platform. (BCC and TCC are usually the +# same unless your are cross-compiling.) +# +TCC = cl.exe -W3 -O2 -DSQLITE_OS_WIN=1 -I. -I$(TOP)\src -I$(TOP)\ext\rtree + +# Define -DNDEBUG to compile without debugging (i.e., for production usage) +# Omitting the define will cause extra debugging code to be inserted and +# includes extra comments when "EXPLAIN stmt" is used. +# +TCC = $(TCC) -DNDEBUG + +# The library that programs using TCL must link against. +# +LIBTCL = tcl85.lib +TCLINCDIR = c:\tcl\include +TCLLIBDIR = c:\tcl\lib + +# This is the command to use for tclsh - normally just "tclsh", but we may +# know the specific version we want to use +# +TCLSH_CMD = tclsh85 + +# Compiler options needed for programs that use the readline() library. +# +READLINE_FLAGS = -DHAVE_READLINE=0 + +# The library that programs using readline() must link against. +# +LIBREADLINE = + +# Should the database engine be compiled threadsafe +# +TCC = $(TCC) -DSQLITE_THREADSAFE=1 + +# Do threads override each others locks by default (1), or do we test (-1) +# +TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1 + +# Any target libraries which libsqlite must be linked against +# +TLIBS = + +# Flags controlling use of the in memory btree implementation +# +# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to +# default to file, 2 to default to memory, and 3 to force temporary +# tables to always be in memory. +# +TEMP_STORE = -DSQLITE_TEMP_STORE=1 + +# Enable/disable loadable extensions, and other optional features +# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). +# The same set of OMIT and ENABLE flags should be passed to the +# LEMON parser generator and the mkkeywordhash tool as well. + +# BEGIN standard options +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 +# END standard options + +# BEGIN required Windows option +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100 +# END required Windows option + +TCC = $(TCC) $(OPT_FEATURE_FLAGS) + +# Add in any optional parameters specified on the make commane line +# ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". +TCC = $(TCC) $(OPTS) + +# Version numbers and release number for the SQLite being compiled. +# +VERSION = 3.7 +VERSION_NUMBER = 3007007 +RELEASE = 3.7.7 + +# libtool compile/link +LTCOMPILE = $(TCC) -Fo$@ +LTLINK = $(TCC) -Fe$@ +LTLIB = lib.exe + +# nawk compatible awk. +NAWK = .\gawk.exe + +# You should not have to change anything below this line +############################################################################### + +USE_AMALGAMATION = 1 + +# Object files for the SQLite library (non-amalgamation). +# +LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ + backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ + callback.lo complete.lo ctime.lo date.lo delete.lo \ + expr.lo fault.lo fkey.lo \ + fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo fts3_porter.lo \ + fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo fts3_write.lo \ + func.lo global.lo hash.lo \ + icu.lo insert.lo journal.lo legacy.lo loadext.lo \ + main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ + memjournal.lo \ + mutex.lo mutex_noop.lo mutex_os2.lo mutex_unix.lo mutex_w32.lo \ + notify.lo opcodes.lo os.lo os_os2.lo os_unix.lo os_win.lo \ + pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ + random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ + table.lo tokenize.lo trigger.lo \ + update.lo util.lo vacuum.lo \ + vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbetrace.lo \ + wal.lo walker.lo where.lo utf.lo vtab.lo + +# Object files for the amalgamation. +# +LIBOBJS1 = sqlite3.lo + +# Determine the real value of LIBOBJ based on the 'configure' script +# +!IF $(USE_AMALGAMATION)==0 +LIBOBJ = $(LIBOBJS0) +!ELSE +LIBOBJ = $(LIBOBJS1) +!ENDIF + +# All of the source code files. +# +SRC = \ + $(TOP)\src\alter.c \ + $(TOP)\src\analyze.c \ + $(TOP)\src\attach.c \ + $(TOP)\src\auth.c \ + $(TOP)\src\backup.c \ + $(TOP)\src\bitvec.c \ + $(TOP)\src\btmutex.c \ + $(TOP)\src\btree.c \ + $(TOP)\src\btree.h \ + $(TOP)\src\btreeInt.h \ + $(TOP)\src\build.c \ + $(TOP)\src\callback.c \ + $(TOP)\src\complete.c \ + $(TOP)\src\ctime.c \ + $(TOP)\src\date.c \ + $(TOP)\src\delete.c \ + $(TOP)\src\expr.c \ + $(TOP)\src\fault.c \ + $(TOP)\src\fkey.c \ + $(TOP)\src\func.c \ + $(TOP)\src\global.c \ + $(TOP)\src\hash.c \ + $(TOP)\src\hash.h \ + $(TOP)\src\hwtime.h \ + $(TOP)\src\insert.c \ + $(TOP)\src\journal.c \ + $(TOP)\src\legacy.c \ + $(TOP)\src\loadext.c \ + $(TOP)\src\main.c \ + $(TOP)\src\malloc.c \ + $(TOP)\src\mem0.c \ + $(TOP)\src\mem1.c \ + $(TOP)\src\mem2.c \ + $(TOP)\src\mem3.c \ + $(TOP)\src\mem5.c \ + $(TOP)\src\memjournal.c \ + $(TOP)\src\mutex.c \ + $(TOP)\src\mutex.h \ + $(TOP)\src\mutex_noop.c \ + $(TOP)\src\mutex_os2.c \ + $(TOP)\src\mutex_unix.c \ + $(TOP)\src\mutex_w32.c \ + $(TOP)\src\notify.c \ + $(TOP)\src\os.c \ + $(TOP)\src\os.h \ + $(TOP)\src\os_common.h \ + $(TOP)\src\os_os2.c \ + $(TOP)\src\os_unix.c \ + $(TOP)\src\os_win.c \ + $(TOP)\src\pager.c \ + $(TOP)\src\pager.h \ + $(TOP)\src\parse.y \ + $(TOP)\src\pcache.c \ + $(TOP)\src\pcache.h \ + $(TOP)\src\pcache1.c \ + $(TOP)\src\pragma.c \ + $(TOP)\src\prepare.c \ + $(TOP)\src\printf.c \ + $(TOP)\src\random.c \ + $(TOP)\src\resolve.c \ + $(TOP)\src\rowset.c \ + $(TOP)\src\select.c \ + $(TOP)\src\status.c \ + $(TOP)\src\shell.c \ + $(TOP)\src\sqlite.h.in \ + $(TOP)\src\sqlite3ext.h \ + $(TOP)\src\sqliteInt.h \ + $(TOP)\src\sqliteLimit.h \ + $(TOP)\src\table.c \ + $(TOP)\src\tclsqlite.c \ + $(TOP)\src\tokenize.c \ + $(TOP)\src\trigger.c \ + $(TOP)\src\utf.c \ + $(TOP)\src\update.c \ + $(TOP)\src\util.c \ + $(TOP)\src\vacuum.c \ + $(TOP)\src\vdbe.c \ + $(TOP)\src\vdbe.h \ + $(TOP)\src\vdbeapi.c \ + $(TOP)\src\vdbeaux.c \ + $(TOP)\src\vdbeblob.c \ + $(TOP)\src\vdbemem.c \ + $(TOP)\src\vdbetrace.c \ + $(TOP)\src\vdbeInt.h \ + $(TOP)\src\vtab.c \ + $(TOP)\src\wal.c \ + $(TOP)\src\wal.h \ + $(TOP)\src\walker.c \ + $(TOP)\src\where.c + +# Source code for extensions +# +SRC = $(SRC) \ + $(TOP)\ext\fts1\fts1.c \ + $(TOP)\ext\fts1\fts1.h \ + $(TOP)\ext\fts1\fts1_hash.c \ + $(TOP)\ext\fts1\fts1_hash.h \ + $(TOP)\ext\fts1\fts1_porter.c \ + $(TOP)\ext\fts1\fts1_tokenizer.h \ + $(TOP)\ext\fts1\fts1_tokenizer1.c +SRC = $(SRC) \ + $(TOP)\ext\fts2\fts2.c \ + $(TOP)\ext\fts2\fts2.h \ + $(TOP)\ext\fts2\fts2_hash.c \ + $(TOP)\ext\fts2\fts2_hash.h \ + $(TOP)\ext\fts2\fts2_icu.c \ + $(TOP)\ext\fts2\fts2_porter.c \ + $(TOP)\ext\fts2\fts2_tokenizer.h \ + $(TOP)\ext\fts2\fts2_tokenizer.c \ + $(TOP)\ext\fts2\fts2_tokenizer1.c +SRC = $(SRC) \ + $(TOP)\ext\fts3\fts3.c \ + $(TOP)\ext\fts3\fts3.h \ + $(TOP)\ext\fts3\fts3Int.h \ + $(TOP)\ext\fts3\fts3_aux.c \ + $(TOP)\ext\fts3\fts3_expr.c \ + $(TOP)\ext\fts3\fts3_hash.c \ + $(TOP)\ext\fts3\fts3_hash.h \ + $(TOP)\ext\fts3\fts3_icu.c \ + $(TOP)\ext\fts3\fts3_porter.c \ + $(TOP)\ext\fts3\fts3_snippet.c \ + $(TOP)\ext\fts3\fts3_tokenizer.h \ + $(TOP)\ext\fts3\fts3_tokenizer.c \ + $(TOP)\ext\fts3\fts3_tokenizer1.c \ + $(TOP)\ext\fts3\fts3_write.c +SRC = $(SRC) \ + $(TOP)\ext\icu\sqliteicu.h \ + $(TOP)\ext\icu\icu.c +SRC = $(SRC) \ + $(TOP)\ext\rtree\rtree.h \ + $(TOP)\ext\rtree\rtree.c + + +# Generated source code files +# +SRC = $(SRC) \ + keywordhash.h \ + opcodes.c \ + opcodes.h \ + parse.c \ + parse.h \ + sqlite3.h + +# Source code to the test files. +# +TESTSRC = \ + $(TOP)\src\test1.c \ + $(TOP)\src\test2.c \ + $(TOP)\src\test3.c \ + $(TOP)\src\test4.c \ + $(TOP)\src\test5.c \ + $(TOP)\src\test6.c \ + $(TOP)\src\test7.c \ + $(TOP)\src\test8.c \ + $(TOP)\src\test9.c \ + $(TOP)\src\test_autoext.c \ + $(TOP)\src\test_async.c \ + $(TOP)\src\test_backup.c \ + $(TOP)\src\test_btree.c \ + $(TOP)\src\test_config.c \ + $(TOP)\src\test_demovfs.c \ + $(TOP)\src\test_devsym.c \ + $(TOP)\src\test_func.c \ + $(TOP)\src\test_fuzzer.c \ + $(TOP)\src\test_hexio.c \ + $(TOP)\src\test_init.c \ + $(TOP)\src\test_intarray.c \ + $(TOP)\src\test_journal.c \ + $(TOP)\src\test_malloc.c \ + $(TOP)\src\test_multiplex.c \ + $(TOP)\src\test_mutex.c \ + $(TOP)\src\test_onefile.c \ + $(TOP)\src\test_osinst.c \ + $(TOP)\src\test_pcache.c \ + $(TOP)\src\test_quota.c \ + $(TOP)\src\test_rtree.c \ + $(TOP)\src\test_schema.c \ + $(TOP)\src\test_server.c \ + $(TOP)\src\test_superlock.c \ + $(TOP)\src\test_syscall.c \ + $(TOP)\src\test_stat.c \ + $(TOP)\src\test_tclvar.c \ + $(TOP)\src\test_thread.c \ + $(TOP)\src\test_vfs.c \ + $(TOP)\src\test_wholenumber.c \ + $(TOP)\src\test_wsd.c \ + $(TOP)\ext\fts3\fts3_term.c \ + $(TOP)\ext\fts3\fts3_test.c + +# Source code to the library files needed by the test fixture +# +TESTSRC2 = \ + $(TOP)\src\attach.c \ + $(TOP)\src\backup.c \ + $(TOP)\src\bitvec.c \ + $(TOP)\src\btree.c \ + $(TOP)\src\build.c \ + $(TOP)\src\ctime.c \ + $(TOP)\src\date.c \ + $(TOP)\src\expr.c \ + $(TOP)\src\func.c \ + $(TOP)\src\insert.c \ + $(TOP)\src\wal.c \ + $(TOP)\src\mem5.c \ + $(TOP)\src\os.c \ + $(TOP)\src\os_os2.c \ + $(TOP)\src\os_unix.c \ + $(TOP)\src\os_win.c \ + $(TOP)\src\pager.c \ + $(TOP)\src\pragma.c \ + $(TOP)\src\prepare.c \ + $(TOP)\src\printf.c \ + $(TOP)\src\random.c \ + $(TOP)\src\pcache.c \ + $(TOP)\src\pcache1.c \ + $(TOP)\src\select.c \ + $(TOP)\src\tokenize.c \ + $(TOP)\src\utf.c \ + $(TOP)\src\util.c \ + $(TOP)\src\vdbeapi.c \ + $(TOP)\src\vdbeaux.c \ + $(TOP)\src\vdbe.c \ + $(TOP)\src\vdbemem.c \ + $(TOP)\src\vdbetrace.c \ + $(TOP)\src\where.c \ + parse.c \ + $(TOP)\ext\fts3\fts3.c \ + $(TOP)\ext\fts3\fts3_aux.c \ + $(TOP)\ext\fts3\fts3_expr.c \ + $(TOP)\ext\fts3\fts3_term.c \ + $(TOP)\ext\fts3\fts3_tokenizer.c \ + $(TOP)\ext\fts3\fts3_write.c \ + $(TOP)\ext\async\sqlite3async.c + +# Header files used by all library source files. +# +HDR = \ + $(TOP)\src\btree.h \ + $(TOP)\src\btreeInt.h \ + $(TOP)\src\hash.h \ + $(TOP)\src\hwtime.h \ + keywordhash.h \ + $(TOP)\src\mutex.h \ + opcodes.h \ + $(TOP)\src\os.h \ + $(TOP)\src\os_common.h \ + $(TOP)\src\pager.h \ + $(TOP)\src\pcache.h \ + parse.h \ + sqlite3.h \ + $(TOP)\src\sqlite3ext.h \ + $(TOP)\src\sqliteInt.h \ + $(TOP)\src\sqliteLimit.h \ + $(TOP)\src\vdbe.h \ + $(TOP)\src\vdbeInt.h + +# Header files used by extensions +# +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\fts1\fts1.h \ + $(TOP)\ext\fts1\fts1_hash.h \ + $(TOP)\ext\fts1\fts1_tokenizer.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\fts2\fts2.h \ + $(TOP)\ext\fts2\fts2_hash.h \ + $(TOP)\ext\fts2\fts2_tokenizer.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\fts3\fts3.h \ + $(TOP)\ext\fts3\fts3Int.h \ + $(TOP)\ext\fts3\fts3_hash.h \ + $(TOP)\ext\fts3\fts3_tokenizer.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\rtree\rtree.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\icu\sqliteicu.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\rtree\sqlite3rtree.h + +# This is the default Makefile target. The objects listed here +# are what get build when you type just "make" with no arguments. +# +all: sqlite3.h libsqlite3.lib sqlite3.exe libtclsqlite3.lib + +Makefile: $(TOP)\Makefile.in + .\config.status + +sqlite3.pc: $(TOP)\sqlite3.pc.in + .\config.status + +libsqlite3.lib: $(LIBOBJ) + $(LTLIB) -OUT:$@ $(LIBOBJ) $(TLIBS) + +libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib + $(LTLIB) /LIBPATH:$(TCLLIBDIR) -OUT:$@ $** $(LIBTCL:tcl=tclstub) $(TLIBS) + +sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib sqlite3.h + $(LTLINK) $(READLINE_FLAGS) \ + $(TOP)\src\shell.c libsqlite3.lib \ + $(LIBREADLINE) $(TLIBS) + +# This target creates a directory named "tsrc" and fills it with +# copies of all of the C source code and header files needed to +# build on the target system. Some of the C source code and header +# files are automatically generated. This target takes care of +# all that automatic generation. +# +.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl + -rmdir /S/Q tsrc + -mkdir tsrc + for %i in ($(SRC)) do copy /Y %i tsrc + del /Q tsrc\sqlite.h.in tsrc\parse.y + $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl vdbe.new + move vdbe.new tsrc\vdbe.c + echo > .target_source + +sqlite3.c: .target_source $(TOP)\tool\mksqlite3c.tcl + $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl + +# Rule to build the amalgamation +# +sqlite3.lo: sqlite3.c + $(LTCOMPILE) $(TEMP_STORE) -c sqlite3.c + +# Rules to build the LEMON compiler generator +# +lempar.c: $(TOP)\src\lempar.c + copy $(TOP)\src\lempar.c . + +lemon.exe: $(TOP)\tool\lemon.c lempar.c + $(BCC) -Fe$@ $(TOP)\tool\lemon.c + +# Rules to build individual *.lo files from generated *.c files. This +# applies to: +# +# parse.lo +# opcodes.lo +# +parse.lo: parse.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c parse.c + +opcodes.lo: opcodes.c + $(LTCOMPILE) $(TEMP_STORE) -c opcodes.c + +# Rules to build individual *.lo files from files in the src directory. +# +alter.lo: $(TOP)\src\alter.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\alter.c + +analyze.lo: $(TOP)\src\analyze.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\analyze.c + +attach.lo: $(TOP)\src\attach.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\attach.c + +auth.lo: $(TOP)\src\auth.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\auth.c + +backup.lo: $(TOP)\src\backup.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\backup.c + +bitvec.lo: $(TOP)\src\bitvec.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\bitvec.c + +btmutex.lo: $(TOP)\src\btmutex.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\btmutex.c + +btree.lo: $(TOP)\src\btree.c $(HDR) $(TOP)\src\pager.h + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\btree.c + +build.lo: $(TOP)\src\build.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\build.c + +callback.lo: $(TOP)\src\callback.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\callback.c + +complete.lo: $(TOP)\src\complete.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\complete.c + +ctime.lo: $(TOP)\src\ctime.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\ctime.c + +date.lo: $(TOP)\src\date.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\date.c + +delete.lo: $(TOP)\src\delete.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\delete.c + +expr.lo: $(TOP)\src\expr.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\expr.c + +fault.lo: $(TOP)\src\fault.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\fault.c + +fkey.lo: $(TOP)\src\fkey.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\fkey.c + +func.lo: $(TOP)\src\func.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\func.c + +global.lo: $(TOP)\src\global.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\global.c + +hash.lo: $(TOP)\src\hash.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\hash.c + +insert.lo: $(TOP)\src\insert.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\insert.c + +journal.lo: $(TOP)\src\journal.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\journal.c + +legacy.lo: $(TOP)\src\legacy.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\legacy.c + +loadext.lo: $(TOP)\src\loadext.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\loadext.c + +main.lo: $(TOP)\src\main.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\main.c + +malloc.lo: $(TOP)\src\malloc.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\malloc.c + +mem0.lo: $(TOP)\src\mem0.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mem0.c + +mem1.lo: $(TOP)\src\mem1.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mem1.c + +mem2.lo: $(TOP)\src\mem2.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mem2.c + +mem3.lo: $(TOP)\src\mem3.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mem3.c + +mem5.lo: $(TOP)\src\mem5.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mem5.c + +memjournal.lo: $(TOP)\src\memjournal.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\memjournal.c + +mutex.lo: $(TOP)\src\mutex.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mutex.c + +mutex_noop.lo: $(TOP)\src\mutex_noop.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mutex_noop.c + +mutex_os2.lo: $(TOP)\src\mutex_os2.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mutex_os2.c + +mutex_unix.lo: $(TOP)\src\mutex_unix.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mutex_unix.c + +mutex_w32.lo: $(TOP)\src\mutex_w32.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mutex_w32.c + +notify.lo: $(TOP)\src\notify.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\notify.c + +pager.lo: $(TOP)\src\pager.c $(HDR) $(TOP)\src\pager.h + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\pager.c + +pcache.lo: $(TOP)\src\pcache.c $(HDR) $(TOP)\src\pcache.h + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\pcache.c + +pcache1.lo: $(TOP)\src\pcache1.c $(HDR) $(TOP)\src\pcache.h + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\pcache1.c + +os.lo: $(TOP)\src\os.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\os.c + +os_unix.lo: $(TOP)\src\os_unix.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\os_unix.c + +os_win.lo: $(TOP)\src\os_win.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\os_win.c + +os_os2.lo: $(TOP)\src\os_os2.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\os_os2.c + +pragma.lo: $(TOP)\src\pragma.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\pragma.c + +prepare.lo: $(TOP)\src\prepare.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\prepare.c + +printf.lo: $(TOP)\src\printf.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\printf.c + +random.lo: $(TOP)\src\random.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\random.c + +resolve.lo: $(TOP)\src\resolve.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\resolve.c + +rowset.lo: $(TOP)\src\rowset.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\rowset.c + +select.lo: $(TOP)\src\select.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\select.c + +status.lo: $(TOP)\src\status.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\status.c + +table.lo: $(TOP)\src\table.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\table.c + +tokenize.lo: $(TOP)\src\tokenize.c keywordhash.h $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\tokenize.c + +trigger.lo: $(TOP)\src\trigger.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\trigger.c + +update.lo: $(TOP)\src\update.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\update.c + +utf.lo: $(TOP)\src\utf.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\utf.c + +util.lo: $(TOP)\src\util.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\util.c + +vacuum.lo: $(TOP)\src\vacuum.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vacuum.c + +vdbe.lo: $(TOP)\src\vdbe.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbe.c + +vdbeapi.lo: $(TOP)\src\vdbeapi.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbeapi.c + +vdbeaux.lo: $(TOP)\src\vdbeaux.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbeaux.c + +vdbeblob.lo: $(TOP)\src\vdbeblob.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbeblob.c + +vdbemem.lo: $(TOP)\src\vdbemem.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbemem.c + +vdbetrace.lo: $(TOP)\src\vdbetrace.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbetrace.c + +vtab.lo: $(TOP)\src\vtab.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vtab.c + +wal.lo: $(TOP)\src\wal.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\wal.c + +walker.lo: $(TOP)\src\walker.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\walker.c + +where.lo: $(TOP)\src\where.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\where.c + +tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) + $(LTCOMPILE) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c + +tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) + $(LTCOMPILE) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c + +tclsqlite-stubs.lo: $(TOP)\src\tclsqlite.c $(HDR) + $(LTCOMPILE) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c + +tclsqlite3.exe: tclsqlite-shell.lo libsqlite3.lib + $(LTLINK) tclsqlite-shell.lo \ + /link /LIBPATH:$(TCLLIBDIR) libsqlite3.lib $(LIBTCL) + +# Rules to build opcodes.c and opcodes.h +# +opcodes.c: opcodes.h $(TOP)\mkopcodec.awk + $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk >opcodes.c + +opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk + type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk >opcodes.h + +# Rules to build parse.c and parse.h - the outputs of lemon. +# +parse.h: parse.c + +parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk + copy $(TOP)\src\parse.y . + del /Q parse.h + .\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y + move parse.h parse.h.temp + $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp >parse.h + +sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION + $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) >sqlite3.h + +mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c + $(BCC) -Femkkeywordhash.exe $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c + +keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe + .\mkkeywordhash.exe >keywordhash.h + + + +# Rules to build the extension objects. +# +icu.lo: $(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c + +fts2.lo: $(TOP)\ext\fts2\fts2.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2.c + +fts2_hash.lo: $(TOP)\ext\fts2\fts2_hash.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_hash.c + +fts2_icu.lo: $(TOP)\ext\fts2\fts2_icu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_icu.c + +fts2_porter.lo: $(TOP)\ext\fts2\fts2_porter.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_porter.c + +fts2_tokenizer.lo: $(TOP)\ext\fts2\fts2_tokenizer.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer.c + +fts2_tokenizer1.lo: $(TOP)\ext\fts2\fts2_tokenizer1.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer1.c + +fts3.lo: $(TOP)\ext\fts3\fts3.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3.c + +fts3_aux.lo: $(TOP)\ext\fts3\fts3_aux.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_aux.c + +fts3_expr.lo: $(TOP)\ext\fts3\fts3_expr.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_expr.c + +fts3_hash.lo: $(TOP)\ext\fts3\fts3_hash.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_hash.c + +fts3_icu.lo: $(TOP)\ext\fts3\fts3_icu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_icu.c + +fts3_snippet.lo: $(TOP)\ext\fts3\fts3_snippet.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_snippet.c + +fts3_porter.lo: $(TOP)\ext\fts3\fts3_porter.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_porter.c + +fts3_tokenizer.lo: $(TOP)\ext\fts3\fts3_tokenizer.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenizer.c + +fts3_tokenizer1.lo: $(TOP)\ext\fts3\fts3_tokenizer1.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenizer1.c + +fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c + +rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c + + +# Rules to build the 'testfixture' application. +# +# If using the amalgamation, use sqlite3.c directly to build the test +# fixture. Otherwise link against libsqlite3.lib. (This distinction is +# necessary because the test fixture requires non-API symbols which are +# hidden when the library is built via the amalgamation). +# +TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE + +TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.lib +TESTFIXTURE_SRC1 = sqlite3.c +!IF $(USE_AMALGAMATION)==0 +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) +!ELSE +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) +!ENDIF + +testfixture.exe: $(TESTFIXTURE_SRC) + $(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \ + -DBUILD_sqlite -I$(TCLINCDIR) \ + $(TESTFIXTURE_SRC) /link /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) + +fulltest: testfixture.exe sqlite3.exe + .\testfixture.exe $(TOP)\test\all.test + +soaktest: testfixture.exe sqlite3.exe + .\testfixture.exe $(TOP)\test\all.test -soak=1 + +test: testfixture.exe sqlite3.exe + .\testfixture.exe $(TOP)\test\veryquick.test + +spaceanal_tcl.h: $(TOP)\tool\spaceanal.tcl + $(NAWK) "/^[^#]/ { gsub(/\\/,\"\\\\\\\\\");gsub(/\\\"/,\"\\\\\\\"\");gsub(/^/,\"\\\"\");gsub(/$$/,\"\\n\\\"\");print }" \ + $(TOP)\tool\spaceanal.tcl >spaceanal_tcl.h + +sqlite3_analyzer.exe: $(TESTFIXTURE_SRC) spaceanal_tcl.h + $(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \ + -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE $(TEMP_STORE) \ + -DBUILD_sqlite -I$(TCLINCDIR) \ + $(TESTFIXTURE_SRC) /link /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) + +clean: + del /Q *.lo *.lib *.obj sqlite3.exe libsqlite3.lib + del /Q sqlite3.h opcodes.c opcodes.h + del /Q lemon.exe lempar.c parse.* + del /Q mkkeywordhash.exe keywordhash.h + -rmdir /Q/S tsrc + del /Q .target_source + del /Q testfixture.exe test.db + del /Q sqlite3.dll sqlite3.lib sqlite3.def + del /Q sqlite3.c + del /Q sqlite3_analyzer.exe spaceanal_tcl.h + +# +# Windows section +# +dll: sqlite3.dll + +sqlite3.def: $(LIBOBJ) + echo $(LIBOBJ) > real.txt + echo $(LIBOBJ) > lib.txt + echo 'EXPORTS' >sqlite3.def + dumpbin /symbols $(LIBOBJ) \ + | $(NAWK) "/SECT.*_sqlite3_/ { sub(/^.* _/,\"\");print }" \ + | sort >>sqlite3.def + +sqlite3.dll: $(LIBOBJ) sqlite3.def + $(TCC) -LD -Fo$@ /DEF:sqlite3.def $(LIBOBJ) diff --git a/manifest b/manifest index 2efad53b59..3b59eb3fd7 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,9 @@ -C Fix\sa\sproblem\swith\sNEAR\squeries\sexecuted\sinside\sa\stransaction\sthat\swrites\sthe\sFTS\stable. -D 2011-06-16T16:06:05.320 +C Add\sMicrosoft\snmake\scompatible\smakefile;\supdate\sa\sfew\stest\scases\sfor\sWindows. +D 2011-06-17T07:07:24.791 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 +F Makefile.msc 20949462727fbf78b219feee331d8e2c65b26fe2 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 3fcdd7fbe3eb282df3978fe77288544543767961 @@ -604,10 +605,10 @@ F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test af02ed0a9cbc0d2a61b8f35171d4d117e588e4ec F test/oserror.test 498d8337e9d15543eb7b004fef8594bf204ff43c -F test/pager1.test 8baf4470b29511503abcaf1f17d16b16462e4d54 +F test/pager1.test 228a831060dab96bc91b03ba2a85cedefd1ab38a F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test 9de4d3e0c59970b4c6cb8dac511fa242f335d8a7 +F test/pagerfault.test 4194b8ea2a5da7958cd155556605ff554e1b065a F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 @@ -945,7 +946,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P a0b43a320e6491de7070966ed7c3ec55fd660a85 -R a91f82245adaecbc51456523e27b5133 -U dan -Z 67955b067165a6636fb02e1c55fe6931 +P 051c756c367837908f6691c0a36108e088c94f99 +R e346269f70235acf3b40c6deb844b2ac +U shaneh +Z 365961d93437a4759811963c7acf64cc diff --git a/manifest.uuid b/manifest.uuid index 999de7fa63..8d9ab1282a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -051c756c367837908f6691c0a36108e088c94f99 \ No newline at end of file +a7590af65f3881cf905b8cac2b87381e89a80271 \ No newline at end of file diff --git a/test/pager1.test b/test/pager1.test index fbdd4b20b9..136ca05f46 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -441,6 +441,8 @@ do_test pager1.4.2.1 { db close tstvfs delete } {} + +if {$::tcl_platform(platform)!="windows"} { do_test pager1.4.2.2 { faultsim_restore_and_reopen execsql { @@ -473,6 +475,7 @@ do_test pager1.4.2.5 { PRAGMA integrity_check; } } {4 ok} +} do_test pager1.4.3.1 { testvfs tstvfs -default 1 @@ -1548,6 +1551,7 @@ do_execsql_test pager1-13.1.1 { UPDATE t1 SET b = a_string(400); } {persist} +if {$::tcl_platform(platform)!="windows"} { # Run transactions of increasing sizes. Eventually, one (or more than one) # of these will write just enough content that one of the old headers created # by the transaction in the block above lies immediately after the content @@ -1570,7 +1574,9 @@ for {set nUp 1} {$nUp<64} {incr nUp} { } {ok} db2 close } +} +if {$::tcl_platform(platform)!="windows"} { # Same test as above. But this time with an index on the table. # do_execsql_test pager1-13.2.1 { @@ -1591,6 +1597,7 @@ for {set nUp 1} {$nUp<64} {incr nUp} { } {ok} db2 close } +} db close tv delete diff --git a/test/pagerfault.test b/test/pagerfault.test index c5c7d3b33a..ced6da3b3c 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -20,6 +20,11 @@ if {[permutation] == "inmemory_journal"} { return } +if {$::tcl_platform(platform)=="windows"} { + finish_test + return +} + set a_string_counter 1 proc a_string {n} { global a_string_counter From 29890211353e2002ec4c21f7e38080f71b003808 Mon Sep 17 00:00:00 2001 From: shaneh Date: Fri, 17 Jun 2011 07:22:09 +0000 Subject: [PATCH 62/69] More updates to the nmake makefile. FossilOrigin-Name: 55bb56d33632eac65bf401dc1f98324b20fb0718 --- Makefile.msc | 8 -------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index a65b16dc39..3e755c71fd 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -422,12 +422,6 @@ EXTHDR = $(EXTHDR) \ # all: sqlite3.h libsqlite3.lib sqlite3.exe libtclsqlite3.lib -Makefile: $(TOP)\Makefile.in - .\config.status - -sqlite3.pc: $(TOP)\sqlite3.pc.in - .\config.status - libsqlite3.lib: $(LIBOBJ) $(LTLIB) -OUT:$@ $(LIBOBJ) $(TLIBS) @@ -854,8 +848,6 @@ clean: dll: sqlite3.dll sqlite3.def: $(LIBOBJ) - echo $(LIBOBJ) > real.txt - echo $(LIBOBJ) > lib.txt echo 'EXPORTS' >sqlite3.def dumpbin /symbols $(LIBOBJ) \ | $(NAWK) "/SECT.*_sqlite3_/ { sub(/^.* _/,\"\");print }" \ diff --git a/manifest b/manifest index 3b59eb3fd7..a810e4f3b7 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Add\sMicrosoft\snmake\scompatible\smakefile;\supdate\sa\sfew\stest\scases\sfor\sWindows. -D 2011-06-17T07:07:24.791 +C More\supdates\sto\sthe\snmake\smakefile. +D 2011-06-17T07:22:09.126 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 20949462727fbf78b219feee331d8e2c65b26fe2 +F Makefile.msc 327a3ba5f00d8abe07b7051d837c8de58a11b6bc F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 3fcdd7fbe3eb282df3978fe77288544543767961 @@ -946,7 +946,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 051c756c367837908f6691c0a36108e088c94f99 -R e346269f70235acf3b40c6deb844b2ac +P a7590af65f3881cf905b8cac2b87381e89a80271 +R c2bb355cb296a58515263b2b4a3bc840 U shaneh -Z 365961d93437a4759811963c7acf64cc +Z 74b5962094639c7853dc6a903a4ac1af diff --git a/manifest.uuid b/manifest.uuid index 8d9ab1282a..10f36aff38 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a7590af65f3881cf905b8cac2b87381e89a80271 \ No newline at end of file +55bb56d33632eac65bf401dc1f98324b20fb0718 \ No newline at end of file From 5e0fb2c64ce7eeab3cdbc155f3baa9699e674370 Mon Sep 17 00:00:00 2001 From: shaneh Date: Fri, 17 Jun 2011 15:54:59 +0000 Subject: [PATCH 63/69] Force almagamation gen scripts to use unix line endings for consistency across platforms. FossilOrigin-Name: a4adc7f78bf43843ee76911baf2f48eb483dd6ac --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/mksqlite3c.tcl | 2 ++ tool/mksqlite3h.tcl | 3 +++ 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a810e4f3b7..f361020875 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\supdates\sto\sthe\snmake\smakefile. -D 2011-06-17T07:22:09.126 +C Force\salmagamation\sgen\sscripts\sto\suse\sunix\sline\sendings\sfor\sconsistency\sacross\splatforms. +D 2011-06-17T15:54:59.552 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -919,8 +919,8 @@ F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c.tcl 623e26cc8c83322e4151d3ad85ac69d41221bae8 -F tool/mksqlite3h.tcl d76c226a5e8e1f3b5f6593bcabe5e98b3b1ec9ff +F tool/mksqlite3c.tcl b23027b185d3e7c7a1803c6f977f68bebd7bc3ec +F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8 F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87 F tool/omittest.tcl b1dd290c1596e0f31fd335160a74ec5dfea3df4a F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -946,7 +946,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P a7590af65f3881cf905b8cac2b87381e89a80271 -R c2bb355cb296a58515263b2b4a3bc840 +P 55bb56d33632eac65bf401dc1f98324b20fb0718 +R 77e80f106180cd2ad766f4c4147bba61 U shaneh -Z 74b5962094639c7853dc6a903a4ac1af +Z bf091bddca3c071d703abfe6c9ac2a5c diff --git a/manifest.uuid b/manifest.uuid index 10f36aff38..971f6c4f4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55bb56d33632eac65bf401dc1f98324b20fb0718 \ No newline at end of file +a4adc7f78bf43843ee76911baf2f48eb483dd6ac \ No newline at end of file diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index fa99f2df7b..b56eaea0c2 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -46,6 +46,8 @@ close $in # of the file. # set out [open sqlite3.c w] +# Force the output to use unix line endings, even on Windows. +# fconfigure $out -translation lf set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1] puts $out [subst \ {/****************************************************************************** diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index 554069c323..f68f61a368 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -65,6 +65,9 @@ close $in set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)} set declpattern {^ *[a-zA-Z][a-zA-Z_0-9 ]+ \**sqlite3_[_a-zA-Z0-9]+\(} +# Force the output to use unix line endings, even on Windows. +fconfigure stdout -translation lf + # Process the src/sqlite.h.in ext/rtree/sqlite3rtree.h files. # foreach file [list $TOP/src/sqlite.h.in $TOP/ext/rtree/sqlite3rtree.h] { From 05fd9aab10f3fe3362beca1a288c03dd1b37ae81 Mon Sep 17 00:00:00 2001 From: shaneh Date: Fri, 17 Jun 2011 15:55:32 +0000 Subject: [PATCH 64/69] Update walro.test script for Windows. FossilOrigin-Name: d6443f8eb30fc3e91e1ad92e2adb595ccf121880 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/walro.test | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f361020875..df596934f9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Force\salmagamation\sgen\sscripts\sto\suse\sunix\sline\sendings\sfor\sconsistency\sacross\splatforms. -D 2011-06-17T15:54:59.552 +C Update\swalro.test\sscript\sfor\sWindows. +D 2011-06-17T15:55:32.570 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -890,7 +890,7 @@ F test/walfault.test 58fce626359c9376fe35101b5c0f2df8040aa839 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 22ddccd073c817ac9ead62b88ac446e8dedc7d2c F test/walnoshm.test a074428046408f4eb5c6a00e09df8cc97ff93317 -F test/walro.test 1f15853383a976ff8bbec78dd44bc15c4e237392 +F test/walro.test 2d5d69e2e99da19ce6faab340330234fc4ca0720 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933 F test/walthread.test a25a393c068a2b42b44333fa3fdaae9072f1617c @@ -946,7 +946,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 55bb56d33632eac65bf401dc1f98324b20fb0718 -R 77e80f106180cd2ad766f4c4147bba61 +P a4adc7f78bf43843ee76911baf2f48eb483dd6ac +R 5c2b696bf2a10ae915d91a0e25d93753 U shaneh -Z bf091bddca3c071d703abfe6c9ac2a5c +Z 0d16be718df2431fa4a5a236a8a45590 diff --git a/manifest.uuid b/manifest.uuid index 971f6c4f4e..73daf7ee40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4adc7f78bf43843ee76911baf2f48eb483dd6ac \ No newline at end of file +d6443f8eb30fc3e91e1ad92e2adb595ccf121880 \ No newline at end of file diff --git a/test/walro.test b/test/walro.test index cf044b75f2..60bdce9e04 100644 --- a/test/walro.test +++ b/test/walro.test @@ -17,12 +17,14 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl set ::testprefix walro +# These tests are only going to work on unix. +# +if {$::tcl_platform(platform) != "unix"} { + finish_test + return +} do_multiclient_test tn { - # These tests are only going to work on unix. - # - if {$tcl_platform(platform) != "unix"} continue - # Do not run tests with the connections in the same process. # if {$tn==2} continue From 6e7850c52076809f8b3b6578f9cf0beb0dfd460a Mon Sep 17 00:00:00 2001 From: shaneh Date: Fri, 17 Jun 2011 15:57:07 +0000 Subject: [PATCH 65/69] Fix a few more issues with the nmake makefile. FossilOrigin-Name: a117005f502482c4529661616cbb26eee1fe75d1 --- Makefile.msc | 211 ++++++++++++++++++++++++++------------------------ manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 118 insertions(+), 107 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 3e755c71fd..4102a33cfe 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -7,16 +7,29 @@ # TOP = . +# Set this non-0 to create and use the SQLite amalgamation file. +# +USE_AMALGAMATION = 1 + # C Compiler and options for use in building executables that # will run on the platform that is doing the build. # BCC = cl.exe -O2 -# C Compile and options for use in building executables that +# C Compile and options for use in building executables that # will run on the target platform. (BCC and TCC are usually the # same unless your are cross-compiling.) # -TCC = cl.exe -W3 -O2 -DSQLITE_OS_WIN=1 -I. -I$(TOP)\src -I$(TOP)\ext\rtree +TCC = cl.exe -W3 -O2 -DSQLITE_OS_WIN=1 -I. -I$(TOP)\src + +# The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in +# any extension header files by default. For non-amalgamation +# builds, we need to make sure the compiler can find these. +# +!IF $(USE_AMALGAMATION)==0 +TCC = $(TCC) -I$(TOP)\ext\fts3 +TCC = $(TCC) -I$(TOP)\ext\rtree +!ENDIF # Define -DNDEBUG to compile without debugging (i.e., for production usage) # Omitting the define will cause extra debugging code to be inserted and @@ -37,11 +50,11 @@ TCLSH_CMD = tclsh85 # Compiler options needed for programs that use the readline() library. # -READLINE_FLAGS = -DHAVE_READLINE=0 +READLINE_FLAGS = -DHAVE_READLINE=0 # The library that programs using readline() must link against. # -LIBREADLINE = +LIBREADLINE = # Should the database engine be compiled threadsafe # @@ -52,8 +65,8 @@ TCC = $(TCC) -DSQLITE_THREADSAFE=1 TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1 # Any target libraries which libsqlite must be linked against -# -TLIBS = +# +TLIBS = # Flags controlling use of the in memory btree implementation # @@ -61,11 +74,11 @@ TLIBS = # default to file, 2 to default to memory, and 3 to force temporary # tables to always be in memory. # -TEMP_STORE = -DSQLITE_TEMP_STORE=1 +TCC = $(TCC) -DSQLITE_TEMP_STORE=1 # Enable/disable loadable extensions, and other optional features -# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). -# The same set of OMIT and ENABLE flags should be passed to the +# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). +# The same set of OMIT and ENABLE flags should be passed to the # LEMON parser generator and the mkkeywordhash tool as well. # BEGIN standard options @@ -101,8 +114,6 @@ NAWK = .\gawk.exe # You should not have to change anything below this line ############################################################################### -USE_AMALGAMATION = 1 - # Object files for the SQLite library (non-amalgamation). # LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ @@ -326,7 +337,7 @@ TESTSRC = \ $(TOP)\src\test_wholenumber.c \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ - $(TOP)\ext\fts3\fts3_test.c + $(TOP)\ext\fts3\fts3_test.c # Source code to the library files needed by the test fixture # @@ -387,10 +398,10 @@ HDR = \ $(TOP)\src\os_common.h \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ - parse.h \ - sqlite3.h \ + parse.h \ + sqlite3.h \ $(TOP)\src\sqlite3ext.h \ - $(TOP)\src\sqliteInt.h \ + $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\vdbe.h \ $(TOP)\src\vdbeInt.h @@ -426,7 +437,7 @@ libsqlite3.lib: $(LIBOBJ) $(LTLIB) -OUT:$@ $(LIBOBJ) $(TLIBS) libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib - $(LTLIB) /LIBPATH:$(TCLLIBDIR) -OUT:$@ $** $(LIBTCL:tcl=tclstub) $(TLIBS) + $(LTLIB) /LIBPATH:$(TCLLIBDIR) -OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS) sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib sqlite3.h $(LTLINK) $(READLINE_FLAGS) \ @@ -454,7 +465,7 @@ sqlite3.c: .target_source $(TOP)\tool\mksqlite3c.tcl # Rule to build the amalgamation # sqlite3.lo: sqlite3.c - $(LTCOMPILE) $(TEMP_STORE) -c sqlite3.c + $(LTCOMPILE) -c sqlite3.c # Rules to build the LEMON compiler generator # @@ -471,222 +482,222 @@ lemon.exe: $(TOP)\tool\lemon.c lempar.c # opcodes.lo # parse.lo: parse.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c parse.c + $(LTCOMPILE) -c parse.c opcodes.lo: opcodes.c - $(LTCOMPILE) $(TEMP_STORE) -c opcodes.c + $(LTCOMPILE) -c opcodes.c # Rules to build individual *.lo files from files in the src directory. # alter.lo: $(TOP)\src\alter.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\alter.c + $(LTCOMPILE) -c $(TOP)\src\alter.c analyze.lo: $(TOP)\src\analyze.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\analyze.c + $(LTCOMPILE) -c $(TOP)\src\analyze.c attach.lo: $(TOP)\src\attach.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\attach.c + $(LTCOMPILE) -c $(TOP)\src\attach.c auth.lo: $(TOP)\src\auth.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\auth.c + $(LTCOMPILE) -c $(TOP)\src\auth.c backup.lo: $(TOP)\src\backup.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\backup.c + $(LTCOMPILE) -c $(TOP)\src\backup.c bitvec.lo: $(TOP)\src\bitvec.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\bitvec.c + $(LTCOMPILE) -c $(TOP)\src\bitvec.c btmutex.lo: $(TOP)\src\btmutex.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\btmutex.c + $(LTCOMPILE) -c $(TOP)\src\btmutex.c btree.lo: $(TOP)\src\btree.c $(HDR) $(TOP)\src\pager.h - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\btree.c + $(LTCOMPILE) -c $(TOP)\src\btree.c build.lo: $(TOP)\src\build.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\build.c + $(LTCOMPILE) -c $(TOP)\src\build.c callback.lo: $(TOP)\src\callback.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\callback.c + $(LTCOMPILE) -c $(TOP)\src\callback.c complete.lo: $(TOP)\src\complete.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\complete.c + $(LTCOMPILE) -c $(TOP)\src\complete.c ctime.lo: $(TOP)\src\ctime.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\ctime.c + $(LTCOMPILE) -c $(TOP)\src\ctime.c date.lo: $(TOP)\src\date.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\date.c + $(LTCOMPILE) -c $(TOP)\src\date.c delete.lo: $(TOP)\src\delete.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\delete.c + $(LTCOMPILE) -c $(TOP)\src\delete.c expr.lo: $(TOP)\src\expr.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\expr.c + $(LTCOMPILE) -c $(TOP)\src\expr.c fault.lo: $(TOP)\src\fault.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\fault.c + $(LTCOMPILE) -c $(TOP)\src\fault.c fkey.lo: $(TOP)\src\fkey.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\fkey.c + $(LTCOMPILE) -c $(TOP)\src\fkey.c func.lo: $(TOP)\src\func.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\func.c + $(LTCOMPILE) -c $(TOP)\src\func.c global.lo: $(TOP)\src\global.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\global.c + $(LTCOMPILE) -c $(TOP)\src\global.c hash.lo: $(TOP)\src\hash.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\hash.c + $(LTCOMPILE) -c $(TOP)\src\hash.c insert.lo: $(TOP)\src\insert.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\insert.c + $(LTCOMPILE) -c $(TOP)\src\insert.c journal.lo: $(TOP)\src\journal.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\journal.c + $(LTCOMPILE) -c $(TOP)\src\journal.c legacy.lo: $(TOP)\src\legacy.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\legacy.c + $(LTCOMPILE) -c $(TOP)\src\legacy.c loadext.lo: $(TOP)\src\loadext.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\loadext.c + $(LTCOMPILE) -c $(TOP)\src\loadext.c main.lo: $(TOP)\src\main.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\main.c + $(LTCOMPILE) -c $(TOP)\src\main.c malloc.lo: $(TOP)\src\malloc.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\malloc.c + $(LTCOMPILE) -c $(TOP)\src\malloc.c mem0.lo: $(TOP)\src\mem0.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mem0.c + $(LTCOMPILE) -c $(TOP)\src\mem0.c mem1.lo: $(TOP)\src\mem1.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mem1.c + $(LTCOMPILE) -c $(TOP)\src\mem1.c mem2.lo: $(TOP)\src\mem2.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mem2.c + $(LTCOMPILE) -c $(TOP)\src\mem2.c mem3.lo: $(TOP)\src\mem3.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mem3.c + $(LTCOMPILE) -c $(TOP)\src\mem3.c mem5.lo: $(TOP)\src\mem5.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mem5.c + $(LTCOMPILE) -c $(TOP)\src\mem5.c memjournal.lo: $(TOP)\src\memjournal.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\memjournal.c + $(LTCOMPILE) -c $(TOP)\src\memjournal.c mutex.lo: $(TOP)\src\mutex.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mutex.c + $(LTCOMPILE) -c $(TOP)\src\mutex.c mutex_noop.lo: $(TOP)\src\mutex_noop.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mutex_noop.c + $(LTCOMPILE) -c $(TOP)\src\mutex_noop.c mutex_os2.lo: $(TOP)\src\mutex_os2.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mutex_os2.c + $(LTCOMPILE) -c $(TOP)\src\mutex_os2.c mutex_unix.lo: $(TOP)\src\mutex_unix.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mutex_unix.c + $(LTCOMPILE) -c $(TOP)\src\mutex_unix.c mutex_w32.lo: $(TOP)\src\mutex_w32.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\mutex_w32.c + $(LTCOMPILE) -c $(TOP)\src\mutex_w32.c notify.lo: $(TOP)\src\notify.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\notify.c + $(LTCOMPILE) -c $(TOP)\src\notify.c pager.lo: $(TOP)\src\pager.c $(HDR) $(TOP)\src\pager.h - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\pager.c + $(LTCOMPILE) -c $(TOP)\src\pager.c pcache.lo: $(TOP)\src\pcache.c $(HDR) $(TOP)\src\pcache.h - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\pcache.c + $(LTCOMPILE) -c $(TOP)\src\pcache.c pcache1.lo: $(TOP)\src\pcache1.c $(HDR) $(TOP)\src\pcache.h - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\pcache1.c + $(LTCOMPILE) -c $(TOP)\src\pcache1.c os.lo: $(TOP)\src\os.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\os.c + $(LTCOMPILE) -c $(TOP)\src\os.c os_unix.lo: $(TOP)\src\os_unix.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\os_unix.c + $(LTCOMPILE) -c $(TOP)\src\os_unix.c os_win.lo: $(TOP)\src\os_win.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\os_win.c + $(LTCOMPILE) -c $(TOP)\src\os_win.c os_os2.lo: $(TOP)\src\os_os2.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\os_os2.c + $(LTCOMPILE) -c $(TOP)\src\os_os2.c pragma.lo: $(TOP)\src\pragma.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\pragma.c + $(LTCOMPILE) -c $(TOP)\src\pragma.c prepare.lo: $(TOP)\src\prepare.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\prepare.c + $(LTCOMPILE) -c $(TOP)\src\prepare.c printf.lo: $(TOP)\src\printf.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\printf.c + $(LTCOMPILE) -c $(TOP)\src\printf.c random.lo: $(TOP)\src\random.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\random.c + $(LTCOMPILE) -c $(TOP)\src\random.c resolve.lo: $(TOP)\src\resolve.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\resolve.c + $(LTCOMPILE) -c $(TOP)\src\resolve.c rowset.lo: $(TOP)\src\rowset.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\rowset.c + $(LTCOMPILE) -c $(TOP)\src\rowset.c select.lo: $(TOP)\src\select.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\select.c + $(LTCOMPILE) -c $(TOP)\src\select.c status.lo: $(TOP)\src\status.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\status.c + $(LTCOMPILE) -c $(TOP)\src\status.c table.lo: $(TOP)\src\table.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\table.c + $(LTCOMPILE) -c $(TOP)\src\table.c tokenize.lo: $(TOP)\src\tokenize.c keywordhash.h $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\tokenize.c + $(LTCOMPILE) -c $(TOP)\src\tokenize.c trigger.lo: $(TOP)\src\trigger.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\trigger.c + $(LTCOMPILE) -c $(TOP)\src\trigger.c update.lo: $(TOP)\src\update.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\update.c + $(LTCOMPILE) -c $(TOP)\src\update.c utf.lo: $(TOP)\src\utf.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\utf.c + $(LTCOMPILE) -c $(TOP)\src\utf.c util.lo: $(TOP)\src\util.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\util.c + $(LTCOMPILE) -c $(TOP)\src\util.c vacuum.lo: $(TOP)\src\vacuum.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vacuum.c + $(LTCOMPILE) -c $(TOP)\src\vacuum.c vdbe.lo: $(TOP)\src\vdbe.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbe.c + $(LTCOMPILE) -c $(TOP)\src\vdbe.c vdbeapi.lo: $(TOP)\src\vdbeapi.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbeapi.c + $(LTCOMPILE) -c $(TOP)\src\vdbeapi.c vdbeaux.lo: $(TOP)\src\vdbeaux.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbeaux.c + $(LTCOMPILE) -c $(TOP)\src\vdbeaux.c vdbeblob.lo: $(TOP)\src\vdbeblob.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbeblob.c + $(LTCOMPILE) -c $(TOP)\src\vdbeblob.c vdbemem.lo: $(TOP)\src\vdbemem.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbemem.c + $(LTCOMPILE) -c $(TOP)\src\vdbemem.c vdbetrace.lo: $(TOP)\src\vdbetrace.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vdbetrace.c + $(LTCOMPILE) -c $(TOP)\src\vdbetrace.c vtab.lo: $(TOP)\src\vtab.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\vtab.c + $(LTCOMPILE) -c $(TOP)\src\vtab.c wal.lo: $(TOP)\src\wal.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\wal.c + $(LTCOMPILE) -c $(TOP)\src\wal.c walker.lo: $(TOP)\src\walker.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\walker.c + $(LTCOMPILE) -c $(TOP)\src\walker.c where.lo: $(TOP)\src\where.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)\src\where.c + $(LTCOMPILE) -c $(TOP)\src\where.c tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(LTCOMPILE) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c @@ -714,8 +725,8 @@ opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk parse.h: parse.c parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk + del /Q parse.y parse.h parse.h.temp copy $(TOP)\src\parse.y . - del /Q parse.h .\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp >parse.h @@ -796,7 +807,7 @@ rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) # hidden when the library is built via the amalgamation). # TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 -TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.lib TESTFIXTURE_SRC1 = sqlite3.c @@ -807,7 +818,7 @@ TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) !ENDIF testfixture.exe: $(TESTFIXTURE_SRC) - $(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \ + $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -DBUILD_sqlite -I$(TCLINCDIR) \ $(TESTFIXTURE_SRC) /link /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) @@ -826,11 +837,11 @@ spaceanal_tcl.h: $(TOP)\tool\spaceanal.tcl sqlite3_analyzer.exe: $(TESTFIXTURE_SRC) spaceanal_tcl.h $(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \ - -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE $(TEMP_STORE) \ + -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE \ -DBUILD_sqlite -I$(TCLINCDIR) \ $(TESTFIXTURE_SRC) /link /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) -clean: +clean: del /Q *.lo *.lib *.obj sqlite3.exe libsqlite3.lib del /Q sqlite3.h opcodes.c opcodes.h del /Q lemon.exe lempar.c parse.* @@ -838,7 +849,7 @@ clean: -rmdir /Q/S tsrc del /Q .target_source del /Q testfixture.exe test.db - del /Q sqlite3.dll sqlite3.lib sqlite3.def + del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def del /Q sqlite3.c del /Q sqlite3_analyzer.exe spaceanal_tcl.h @@ -847,11 +858,11 @@ clean: # dll: sqlite3.dll -sqlite3.def: $(LIBOBJ) - echo 'EXPORTS' >sqlite3.def - dumpbin /symbols $(LIBOBJ) \ - | $(NAWK) "/SECT.*_sqlite3_/ { sub(/^.* _/,\"\");print }" \ +sqlite3.def: libsqlite3.lib + echo EXPORTS >sqlite3.def + dumpbin /all libsqlite3.lib \ + | $(NAWK) "/ 1 _sqlite3_/ { sub(/^.* _/,\"\");print }" \ | sort >>sqlite3.def sqlite3.dll: $(LIBOBJ) sqlite3.def - $(TCC) -LD -Fo$@ /DEF:sqlite3.def $(LIBOBJ) + link /DLL /OUT:$@ /DEF:sqlite3.def $(LIBOBJ) diff --git a/manifest b/manifest index df596934f9..2adc201993 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Update\swalro.test\sscript\sfor\sWindows. -D 2011-06-17T15:55:32.570 +C Fix\sa\sfew\smore\sissues\swith\sthe\snmake\smakefile. +D 2011-06-17T15:57:07.878 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 327a3ba5f00d8abe07b7051d837c8de58a11b6bc +F Makefile.msc b84e4a1636babdf33ba1c339da6ae4e30d61bdce F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 3fcdd7fbe3eb282df3978fe77288544543767961 @@ -946,7 +946,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P a4adc7f78bf43843ee76911baf2f48eb483dd6ac -R 5c2b696bf2a10ae915d91a0e25d93753 +P d6443f8eb30fc3e91e1ad92e2adb595ccf121880 +R efe888bea588d8664ec1e53b3ca0b8eb U shaneh -Z 0d16be718df2431fa4a5a236a8a45590 +Z 8a36b3551ed8c68b140a2528bd77bdc7 diff --git a/manifest.uuid b/manifest.uuid index 73daf7ee40..8b7519e9bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6443f8eb30fc3e91e1ad92e2adb595ccf121880 \ No newline at end of file +a117005f502482c4529661616cbb26eee1fe75d1 \ No newline at end of file From 10fc4d8c450d42d4bb2c9c7a60e061185fe8263e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Jun 2011 16:04:39 +0000 Subject: [PATCH 66/69] Add a missing declaration to fts3Int.h. FossilOrigin-Name: 3bfd4466f50711eb71d1a13231025ff4e1e76246 --- ext/fts3/fts3Int.h | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 5d3d22a57a..980b52c913 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -509,6 +509,7 @@ int sqlite3Fts3MsrIncrNext( Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); diff --git a/manifest b/manifest index 2adc201993..6f625d03e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfew\smore\sissues\swith\sthe\snmake\smakefile. -D 2011-06-17T15:57:07.878 +C Add\sa\smissing\sdeclaration\sto\sfts3Int.h. +D 2011-06-17T16:04:39.347 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -64,7 +64,7 @@ F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c 78b02b5f0195e397c4239ef9213e5506b7d3fa97 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h df761492ae2308d3d56123907ca29cdf9bdd3748 +F ext/fts3/fts3Int.h 974cf471fae5fed8ad87069cd86c1fe5a9bf6f9c F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb @@ -946,7 +946,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P d6443f8eb30fc3e91e1ad92e2adb595ccf121880 -R efe888bea588d8664ec1e53b3ca0b8eb -U shaneh -Z 8a36b3551ed8c68b140a2528bd77bdc7 +P a117005f502482c4529661616cbb26eee1fe75d1 +R faf357353d86a944921df5db701e63d3 +U dan +Z d361a7bc828cbb05e3ac27317d622fe4 diff --git a/manifest.uuid b/manifest.uuid index 8b7519e9bd..2c02eba3f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a117005f502482c4529661616cbb26eee1fe75d1 \ No newline at end of file +3bfd4466f50711eb71d1a13231025ff4e1e76246 \ No newline at end of file From c5f21892a8676763632d2d02c151e611c2fb3797 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Jun 2011 17:37:31 +0000 Subject: [PATCH 67/69] Avoid loading doclists for infrequent terms that are part of phrases twice. FossilOrigin-Name: 8f939723f742329cedba8930f71dff42004f3d0d --- ext/fts3/fts3.c | 123 ++++++++++++++++++++++++++++----------------- ext/fts3/fts3Int.h | 2 +- manifest | 14 +++--- manifest.uuid | 2 +- 4 files changed, 85 insertions(+), 56 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index f2c907c8c0..b67cb8c5a1 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3182,6 +3182,8 @@ static void fts3EvalAllocateReaders( return; } } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; }else{ *pnOr += (pExpr->eType==FTSQUERY_OR); fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); @@ -3190,6 +3192,60 @@ static void fts3EvalAllocateReaders( } } +static void fts3EvalPhraseMergeToken( + Fts3Table *pTab, + Fts3Phrase *p, + int iToken, + char *pList, + int nList +){ + assert( iToken!=p->iDoclistToken ); + + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; + } + + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } + + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); + } + + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; + + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } + + fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; +} + static int fts3EvalPhraseLoad( Fts3Cursor *pCsr, Fts3Phrase *p @@ -3198,47 +3254,21 @@ static int fts3EvalPhraseLoad( int iToken; int rc = SQLITE_OK; - char *aDoclist = 0; - int nDoclist = 0; - int iPrev = -1; - for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ Fts3PhraseToken *pToken = &p->aToken[iToken]; - assert( pToken->pSegcsr || pToken->pDeferred ); + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); - if( pToken->pDeferred==0 ){ + if( pToken->pSegcsr ){ int nThis = 0; char *pThis = 0; rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); if( rc==SQLITE_OK ){ - if( pThis==0 ){ - sqlite3_free(aDoclist); - aDoclist = 0; - nDoclist = 0; - break; - }else if( aDoclist==0 ){ - aDoclist = pThis; - nDoclist = nThis; - }else{ - assert( iPrev>=0 ); - fts3DoclistPhraseMerge(pTab->bDescIdx, - iToken-iPrev, aDoclist, nDoclist, pThis, &nThis - ); - sqlite3_free(aDoclist); - aDoclist = pThis; - nDoclist = nThis; - } - iPrev = iToken; + fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); } } + assert( pToken->pSegcsr==0 ); } - if( rc==SQLITE_OK ){ - p->doclist.aAll = aDoclist; - p->doclist.nAll = nDoclist; - }else{ - sqlite3_free(aDoclist); - } return rc; } @@ -3246,7 +3276,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ int iToken; int rc = SQLITE_OK; - int nMaxUndeferred = -1; + int nMaxUndeferred = pPhrase->iDoclistToken; char *aPoslist = 0; int nPoslist = 0; int iPrev = -1; @@ -3291,8 +3321,6 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ } } iPrev = iToken; - }else{ - nMaxUndeferred = iToken; } } @@ -3351,9 +3379,11 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ Fts3PhraseToken *pFirst = &p->aToken[0]; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - assert( p->doclist.aAll==0 ); - if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 - && pFirst->pSegcsr && pFirst->pSegcsr->bLookup + if( pCsr->bDesc==pTab->bDescIdx + && bOptOk==1 + && p->nToken==1 + && pFirst->pSegcsr + && pFirst->pSegcsr->bLookup ){ /* Use the incremental approach. */ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); @@ -3524,13 +3554,14 @@ static void fts3EvalStartReaders( } } - typedef struct Fts3TokenAndCost Fts3TokenAndCost; struct Fts3TokenAndCost { - Fts3PhraseToken *pToken; - Fts3Expr *pRoot; + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; int nOvfl; - int iCol; + int iCol; /* The column the token must match */ }; static void fts3EvalTokenCosts( @@ -3547,6 +3578,8 @@ static void fts3EvalTokenCosts( int i; for(i=0; *pRc==SQLITE_OK && inToken; i++){ Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; pTC->pRoot = pRoot; pTC->pToken = &pPhrase->aToken[i]; pTC->iCol = pPhrase->iColumn; @@ -3659,19 +3692,15 @@ static int fts3EvalSelectDeferred( if( pTC->nOvfl ){ nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; }else{ - /* TODO: Fix this so that the doclist need not be read twice. */ Fts3PhraseToken *pToken = pTC->pToken; int nList = 0; char *pList = 0; rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ nDocEst = fts3DoclistCountDocids(1, pList, nList); - } - sqlite3_free(pList); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3TermSegReaderCursor(pCsr, - pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr - ); + fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); } } }else{ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 980b52c913..acdaf3d5bb 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -306,7 +306,6 @@ struct Fts3PhraseToken { /* Variables above this point are populated when the expression is ** parsed (by code in fts3_expr.c). Below this point the variables are ** used when evaluating the expression. */ - int bFulltext; /* True if full-text index was used */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ }; @@ -315,6 +314,7 @@ struct Fts3Phrase { /* Cache of doclist for this phrase. */ Fts3Doclist doclist; int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; /* Variables below this point are populated by fts3_expr.c when parsing ** a MATCH expression. Everything above is part of the evaluation phase. diff --git a/manifest b/manifest index 6f625d03e0..155d28276b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\sdeclaration\sto\sfts3Int.h. -D 2011-06-17T16:04:39.347 +C Avoid\sloading\sdoclists\sfor\sinfrequent\sterms\sthat\sare\spart\sof\sphrases\stwice. +D 2011-06-17T17:37:31.284 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,9 +62,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 78b02b5f0195e397c4239ef9213e5506b7d3fa97 +F ext/fts3/fts3.c f919a7966426e539b3f39f696bc94269e3726033 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 974cf471fae5fed8ad87069cd86c1fe5a9bf6f9c +F ext/fts3/fts3Int.h 8ece4390eb44e7179bb05c59d40f447663f5c077 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb @@ -946,7 +946,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P a117005f502482c4529661616cbb26eee1fe75d1 -R faf357353d86a944921df5db701e63d3 +P 3bfd4466f50711eb71d1a13231025ff4e1e76246 +R cd3d6544b7cd834ce139927827bd3fe8 U dan -Z d361a7bc828cbb05e3ac27317d622fe4 +Z fa9494c386a1e1a2787b09d7f28a7b96 diff --git a/manifest.uuid b/manifest.uuid index 2c02eba3f4..cda471ef25 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3bfd4466f50711eb71d1a13231025ff4e1e76246 \ No newline at end of file +8f939723f742329cedba8930f71dff42004f3d0d \ No newline at end of file From 603e426f9544ce64f54424d9dacdc8c98df3b185 Mon Sep 17 00:00:00 2001 From: shaneh Date: Fri, 17 Jun 2011 18:52:07 +0000 Subject: [PATCH 68/69] Fix a header dependency in nmake Makefile. FossilOrigin-Name: 54492212af6c4be5d5546b370398ef771c16f430 --- Makefile.msc | 26 +++++++++++--------------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 4102a33cfe..68a66bcb45 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1,5 +1,5 @@ # -# nmake Makefile for SQLITE +# nmake Makefile for SQLite # # The toplevel directory of the source tree. This is the directory @@ -11,6 +11,12 @@ TOP = . # USE_AMALGAMATION = 1 +# Version numbers and release number for the SQLite being compiled. +# +VERSION = 3.7 +VERSION_NUMBER = 3007007 +RELEASE = 3.7.7 + # C Compiler and options for use in building executables that # will run on the platform that is doing the build. # @@ -97,12 +103,6 @@ TCC = $(TCC) $(OPT_FEATURE_FLAGS) # ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". TCC = $(TCC) $(OPTS) -# Version numbers and release number for the SQLite being compiled. -# -VERSION = 3.7 -VERSION_NUMBER = 3007007 -RELEASE = 3.7.7 - # libtool compile/link LTCOMPILE = $(TCC) -Fo$@ LTLINK = $(TCC) -Fe$@ @@ -379,7 +379,6 @@ TESTSRC2 = \ $(TOP)\ext\fts3\fts3.c \ $(TOP)\ext\fts3\fts3_aux.c \ $(TOP)\ext\fts3\fts3_expr.c \ - $(TOP)\ext\fts3\fts3_term.c \ $(TOP)\ext\fts3\fts3_tokenizer.c \ $(TOP)\ext\fts3\fts3_write.c \ $(TOP)\ext\async\sqlite3async.c @@ -431,7 +430,7 @@ EXTHDR = $(EXTHDR) \ # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # -all: sqlite3.h libsqlite3.lib sqlite3.exe libtclsqlite3.lib +all: libsqlite3.lib sqlite3.exe libtclsqlite3.lib libsqlite3.lib: $(LIBOBJ) $(LTLIB) -OUT:$@ $(LIBOBJ) $(TLIBS) @@ -705,9 +704,6 @@ tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(LTCOMPILE) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c -tclsqlite-stubs.lo: $(TOP)\src\tclsqlite.c $(HDR) - $(LTCOMPILE) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c - tclsqlite3.exe: tclsqlite-shell.lo libsqlite3.lib $(LTLINK) tclsqlite-shell.lo \ /link /LIBPATH:$(TCLLIBDIR) libsqlite3.lib $(LIBTCL) @@ -817,7 +813,7 @@ TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) !ENDIF -testfixture.exe: $(TESTFIXTURE_SRC) +testfixture.exe: $(TESTFIXTURE_SRC) $(HDR) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -DBUILD_sqlite -I$(TCLINCDIR) \ $(TESTFIXTURE_SRC) /link /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) @@ -848,10 +844,10 @@ clean: del /Q mkkeywordhash.exe keywordhash.h -rmdir /Q/S tsrc del /Q .target_source - del /Q testfixture.exe test.db + del /Q testfixture.exe testfixture.exp test.db del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def del /Q sqlite3.c - del /Q sqlite3_analyzer.exe spaceanal_tcl.h + del /Q sqlite3_analyzer.exe sqlite3_analyzer.exp spaceanal_tcl.h # # Windows section diff --git a/manifest b/manifest index 155d28276b..ce32a1ff36 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Avoid\sloading\sdoclists\sfor\sinfrequent\sterms\sthat\sare\spart\sof\sphrases\stwice. -D 2011-06-17T17:37:31.284 +C Fix\sa\sheader\sdependency\sin\snmake\sMakefile. +D 2011-06-17T18:52:07.172 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc b84e4a1636babdf33ba1c339da6ae4e30d61bdce +F Makefile.msc ce73810c83d4bd202deb59d547a5df4a5cfef7c9 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 3fcdd7fbe3eb282df3978fe77288544543767961 @@ -946,7 +946,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 3bfd4466f50711eb71d1a13231025ff4e1e76246 -R cd3d6544b7cd834ce139927827bd3fe8 -U dan -Z fa9494c386a1e1a2787b09d7f28a7b96 +P 8f939723f742329cedba8930f71dff42004f3d0d +R d5bee1c0649d20e80531413a35f05bd9 +U shaneh +Z 03c84e62572562693578ac95748470fd diff --git a/manifest.uuid b/manifest.uuid index cda471ef25..769a2d893d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f939723f742329cedba8930f71dff42004f3d0d \ No newline at end of file +54492212af6c4be5d5546b370398ef771c16f430 \ No newline at end of file From d80138af2c42ea2f1cd9627823281f6b1e20525e Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 19 Jun 2011 21:17:35 +0000 Subject: [PATCH 69/69] Do not run test tkt-2d1a5c67d.test in the inmemory_journal permutation since that test requires WAL mode which does not work with inmemory_journal. FossilOrigin-Name: 228c43c726e637daadc0c9b5a8b24243f239b1cf --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/permutations.test | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ce32a1ff36..1d5ddc2579 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sheader\sdependency\sin\snmake\sMakefile. -D 2011-06-17T18:52:07.172 +C Do\snot\srun\stest\stkt-2d1a5c67d.test\sin\sthe\sinmemory_journal\spermutation\nsince\sthat\stest\srequires\sWAL\smode\swhich\sdoes\snot\swork\swith\sinmemory_journal. +D 2011-06-19T21:17:35.711 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -615,7 +615,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 3a0d43a3e92f74f49be42ddc580e9b1ec361ba0e +F test/permutations.test 1e8892ebf1bd6e9e8036f4841c72a91bf72da74a F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -946,7 +946,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 8f939723f742329cedba8930f71dff42004f3d0d -R d5bee1c0649d20e80531413a35f05bd9 -U shaneh -Z 03c84e62572562693578ac95748470fd +P 54492212af6c4be5d5546b370398ef771c16f430 +R dd08c420fef14e9aaef20168e7aef1b4 +U drh +Z ad3555b405ce175e6eb80427a259c9c5 diff --git a/manifest.uuid b/manifest.uuid index 769a2d893d..568bed9d24 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54492212af6c4be5d5546b370398ef771c16f430 \ No newline at end of file +228c43c726e637daadc0c9b5a8b24243f239b1cf \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 9c95296e39..22e2189d98 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -529,7 +529,7 @@ test_suite "inmemory_journal" -description { stmt.test # WAL mode is different. - wal* + wal* tkt-2d1a5c67d.test }] ifcapable mem3 {