From d9f9441d7d4435bcd714b2cbda597c6da45c535a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Sep 2014 03:22:27 +0000 Subject: [PATCH 01/25] Disable shared memory operations using the unix-nolock VFS. FossilOrigin-Name: 10a6e510497b471d67ac3dfb19ff256a7d18adf4 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/os_unix.c | 28 ++++++++++++++++++---------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index ddf26d66a1..b51e6af8f7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correctly\shandle\san\sORDER\sBY\sclause\son\san\souter\squery\swhen\sapplying\nthe\scompound-subquery\sflattening\soptimization.\s\sTicket\s[89398880bcfff].\nAlso\sadd\sthe\sSQLITE_ENABLE_SELECTTRACE\soption\sfor\sadditional\sdebugging\nand\sanalysis\sinformation\sabout\sselect\sstatement\sprocessing. -D 2014-09-21T22:31:52.593 +C Disable\sshared\smemory\soperations\susing\sthe\sunix-nolock\sVFS. +D 2014-09-22T03:22:27.824 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -212,7 +212,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 9096a1b1449182e67e759f59994eee04113bc587 +F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c caab007743821d96752597c9cfd7351654697b06 @@ -1199,8 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 59e2c9df02d7e988c5ad44c560ead1e5288b12e7 9683e001ed38b41979220eef0bdfcb54df5f3191 -R df45958fe8f93552efb788581c98f0f7 -T +closed 9683e001ed38b41979220eef0bdfcb54df5f3191 +P d5880abd63c83c88e135257373afa0a3fd88297e +R 78a2249f9b04bb5d20ea2c24b3aed300 U drh -Z d861c46be738b6f104b52fa4441ce204 +Z 8d87b3613ef02056468b7e3fc8779d9f diff --git a/manifest.uuid b/manifest.uuid index 7cf6d685c3..87021972ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5880abd63c83c88e135257373afa0a3fd88297e \ No newline at end of file +10a6e510497b471d67ac3dfb19ff256a7d18adf4 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index fcd9e72d06..a9344ee830 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4951,7 +4951,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ ** * An I/O method finder function called FINDER that returns a pointer ** to the METHOD object in the previous bullet. */ -#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK) \ +#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK, SHMMAP) \ static const sqlite3_io_methods METHOD = { \ VERSION, /* iVersion */ \ CLOSE, /* xClose */ \ @@ -4966,7 +4966,7 @@ static const sqlite3_io_methods METHOD = { \ unixFileControl, /* xFileControl */ \ unixSectorSize, /* xSectorSize */ \ unixDeviceCharacteristics, /* xDeviceCapabilities */ \ - unixShmMap, /* xShmMap */ \ + SHMMAP, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmUnmap, /* xShmUnmap */ \ @@ -4992,7 +4992,8 @@ IOMETHODS( unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + unixShmMap /* xShmMap method */ ) IOMETHODS( nolockIoFinder, /* Finder function name */ @@ -5001,7 +5002,8 @@ IOMETHODS( nolockClose, /* xClose method */ nolockLock, /* xLock method */ nolockUnlock, /* xUnlock method */ - nolockCheckReservedLock /* xCheckReservedLock method */ + nolockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) IOMETHODS( dotlockIoFinder, /* Finder function name */ @@ -5010,7 +5012,8 @@ IOMETHODS( dotlockClose, /* xClose method */ dotlockLock, /* xLock method */ dotlockUnlock, /* xUnlock method */ - dotlockCheckReservedLock /* xCheckReservedLock method */ + dotlockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS @@ -5021,7 +5024,8 @@ IOMETHODS( flockClose, /* xClose method */ flockLock, /* xLock method */ flockUnlock, /* xUnlock method */ - flockCheckReservedLock /* xCheckReservedLock method */ + flockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5033,7 +5037,8 @@ IOMETHODS( semClose, /* xClose method */ semLock, /* xLock method */ semUnlock, /* xUnlock method */ - semCheckReservedLock /* xCheckReservedLock method */ + semCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5045,7 +5050,8 @@ IOMETHODS( afpClose, /* xClose method */ afpLock, /* xLock method */ afpUnlock, /* xUnlock method */ - afpCheckReservedLock /* xCheckReservedLock method */ + afpCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5070,7 +5076,8 @@ IOMETHODS( proxyClose, /* xClose method */ proxyLock, /* xLock method */ proxyUnlock, /* xUnlock method */ - proxyCheckReservedLock /* xCheckReservedLock method */ + proxyCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5083,7 +5090,8 @@ IOMETHODS( unixClose, /* xClose method */ unixLock, /* xLock method */ nfsUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif From 7e07433fe72598b932a2115d2774a98e12f50789 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Sep 2014 14:30:51 +0000 Subject: [PATCH 02/25] Tune the query planner to be more aggressive about using automatic indexes on views and subqueries for which there is not opportunity to declare a persistent schema index. FossilOrigin-Name: 41de1643bfc9ae25e20790d707e2789b665baa2b --- manifest | 14 +++---- manifest.uuid | 2 +- src/where.c | 14 +++++-- test/autoindex1.test | 97 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b51e6af8f7..926eedc817 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sshared\smemory\soperations\susing\sthe\sunix-nolock\sVFS. -D 2014-09-22T03:22:27.824 +C Tune\sthe\squery\splanner\sto\sbe\smore\saggressive\sabout\susing\sautomatic\sindexes\non\sviews\sand\ssubqueries\sfor\swhich\sthere\sis\snot\sopportunity\sto\sdeclare\sa\npersistent\sschema\sindex. +D 2014-09-22T14:30:51.911 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 0888567c0e01a41b6001647e333f8ccfd3ae7d36 +F src/where.c 3f859ecfada8643ce8255f20d481b5a3e0921662 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -341,7 +341,7 @@ F test/auth.test 855233ef26eb3601b6886567ea4e326c72959360 F test/auth2.test 264c6af53cad9aba5218c68bbe18036e39007bfa F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 -F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 +F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de F test/autoindex3.test 8254f689c3241081fad52b7bea18ba53e07e14a2 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d5880abd63c83c88e135257373afa0a3fd88297e -R 78a2249f9b04bb5d20ea2c24b3aed300 +P 10a6e510497b471d67ac3dfb19ff256a7d18adf4 +R f6ce79b0bd0352edf4a2c65db2a400a7 U drh -Z 8d87b3613ef02056468b7e3fc8779d9f +Z b1246ec5b6570c51d45a58350d46840c diff --git a/manifest.uuid b/manifest.uuid index 87021972ad..a03afa027d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -10a6e510497b471d67ac3dfb19ff256a7d18adf4 \ No newline at end of file +41de1643bfc9ae25e20790d707e2789b665baa2b \ No newline at end of file diff --git a/src/where.c b/src/where.c index 318065d486..d1862b499a 100644 --- a/src/where.c +++ b/src/where.c @@ -4718,9 +4718,17 @@ static int whereLoopAddBtree( pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; /* TUNING: One-time cost for computing the automatic index is - ** approximately 7*N*log2(N) where N is the number of rows in - ** the table being indexed. */ - pNew->rSetup = rLogSize + rSize + 28; assert( 28==sqlite3LogEst(7) ); + ** estimated to be X*N*log2(N) where N is the number of rows in + ** the table being indexed and where X is 7 (LogEst=28) for normal + ** tables or 1.375 (LogEst=4) for views and subqueries. The value + ** of X is smaller for views and subqueries so that the query planner + ** will be more aggressive about generating automatic indexes for + ** those objects, since there is no opportunity to add schema + ** indexes on subqueries and views. */ + pNew->rSetup = rLogSize + rSize + 4; + if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){ + pNew->rSetup += 24; + } ApplyCostMultiplier(pNew->rSetup, pTab->costMult); /* TUNING: Each index lookup yields 20 rows in the table. This ** is more than the usual guess of 10 rows, since we have no way diff --git a/test/autoindex1.test b/test/autoindex1.test index 6cb0ab146a..bcde5bc2e7 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -413,4 +413,101 @@ do_execsql_test autoindex1-801 { WHERE mimetypes._id=10 AND data14 IS NOT NULL; } {/SEARCH TABLE data .*SEARCH TABLE raw_contacts/} +# Another test case from an important user of SQLite. The key feature of +# this test is that the "aggindex" subquery should make use of an +# automatic index. If it does, the query is fast. If it does not, the +# query is deathly slow. It worked OK in 3.7.17 but started going slow +# with version 3.8.0. The problem was fixed for 3.8.7 by reducing the +# cost estimate for automatic indexes on views and subqueries. +# +db close +forcedelete test.db +sqlite3 db test.db +do_execsql_test autoindex1-900 { + CREATE TABLE messages (ROWID INTEGER PRIMARY KEY AUTOINCREMENT, message_id, document_id BLOB, in_reply_to, remote_id INTEGER, sender INTEGER, subject_prefix, subject INTEGER, date_sent INTEGER, date_received INTEGER, date_created INTEGER, date_last_viewed INTEGER, mailbox INTEGER, remote_mailbox INTEGER, original_mailbox INTEGER, flags INTEGER, read, flagged, size INTEGER, color, encoding, type INTEGER, pad, conversation_id INTEGER DEFAULT -1, snippet TEXT DEFAULT NULL, fuzzy_ancestor INTEGER DEFAULT NULL, automated_conversation INTEGER DEFAULT 0, root_status INTEGER DEFAULT -1, conversation_position INTEGER DEFAULT -1); + CREATE INDEX date_index ON messages(date_received); + CREATE INDEX date_last_viewed_index ON messages(date_last_viewed); + CREATE INDEX date_created_index ON messages(date_created); + CREATE INDEX message_message_id_mailbox_index ON messages(message_id, mailbox); + CREATE INDEX message_document_id_index ON messages(document_id); + CREATE INDEX message_read_index ON messages(read); + CREATE INDEX message_flagged_index ON messages(flagged); + CREATE INDEX message_mailbox_index ON messages(mailbox, date_received); + CREATE INDEX message_remote_mailbox_index ON messages(remote_mailbox, remote_id); + CREATE INDEX message_type_index ON messages(type); + CREATE INDEX message_conversation_id_conversation_position_index ON messages(conversation_id, conversation_position); + CREATE INDEX message_fuzzy_ancestor_index ON messages(fuzzy_ancestor); + CREATE INDEX message_subject_fuzzy_ancestor_index ON messages(subject, fuzzy_ancestor); + CREATE INDEX message_sender_subject_automated_conversation_index ON messages(sender, subject, automated_conversation); + CREATE INDEX message_sender_index ON messages(sender); + CREATE INDEX message_root_status ON messages(root_status); + CREATE TABLE subjects (ROWID INTEGER PRIMARY KEY, subject COLLATE RTRIM, normalized_subject COLLATE RTRIM); + CREATE INDEX subject_subject_index ON subjects(subject); + CREATE INDEX subject_normalized_subject_index ON subjects(normalized_subject); + CREATE TABLE addresses (ROWID INTEGER PRIMARY KEY, address COLLATE NOCASE, comment, UNIQUE(address, comment)); + CREATE INDEX addresses_address_index ON addresses(address); + CREATE TABLE mailboxes (ROWID INTEGER PRIMARY KEY, url UNIQUE, total_count INTEGER DEFAULT 0, unread_count INTEGER DEFAULT 0, unseen_count INTEGER DEFAULT 0, deleted_count INTEGER DEFAULT 0, unread_count_adjusted_for_duplicates INTEGER DEFAULT 0, change_identifier, source INTEGER, alleged_change_identifier); + CREATE INDEX mailboxes_source_index ON mailboxes(source); + CREATE TABLE labels (ROWID INTEGER PRIMARY KEY, message_id INTEGER NOT NULL, mailbox_id INTEGER NOT NULL, UNIQUE(message_id, mailbox_id)); + CREATE INDEX labels_message_id_mailbox_id_index ON labels(message_id, mailbox_id); + CREATE INDEX labels_mailbox_id_index ON labels(mailbox_id); + + explain query plan + SELECT messages.ROWID, + messages.message_id, + messages.remote_id, + messages.date_received, + messages.date_sent, + messages.flags, + messages.size, + messages.color, + messages.date_last_viewed, + messages.subject_prefix, + subjects.subject, + sender.comment, + sender.address, + NULL, + messages.mailbox, + messages.original_mailbox, + NULL, + NULL, + messages.type, + messages.document_id, + sender, + NULL, + messages.conversation_id, + messages.conversation_position, + agglabels.labels + FROM mailboxes AS mailbox + JOIN messages ON mailbox.ROWID = messages.mailbox + LEFT OUTER JOIN subjects ON messages.subject = subjects.ROWID + LEFT OUTER JOIN addresses AS sender ON messages.sender = sender.ROWID + LEFT OUTER JOIN ( + SELECT message_id, group_concat(mailbox_id) as labels + FROM labels GROUP BY message_id + ) AS agglabels ON messages.ROWID = agglabels.message_id + WHERE (mailbox.url = 'imap://email.app@imap.gmail.com/%5BGmail%5D/All%20Mail') + AND (messages.ROWID IN ( + SELECT labels.message_id + FROM labels JOIN mailboxes ON labels.mailbox_id = mailboxes.ROWID + WHERE mailboxes.url = 'imap://email.app@imap.gmail.com/INBOX')) + AND messages.mailbox in (6,12,18,24,30,36,42,1,7,13,19,25,31,37,43,2,8, + 14,20,26,32,38,3,9,15,21,27,33,39,4,10,16,22,28, + 34,40,5,11,17,23,35,41) + ORDER BY date_received DESC; +} {/agglabels USING AUTOMATIC COVERING INDEX/} + +# A test case for VIEWs +# +do_execsql_test autoindex1-901 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z); + CREATE TABLE t2(a, b); + CREATE VIEW agg2 AS SELECT a, sum(b) AS m FROM t2 GROUP BY a; + EXPLAIN QUERY PLAN + SELECT t1.z, agg2.m + FROM t1 JOIN agg2 ON t1.y=agg2.m + WHERE t1.x IN (1,2,3); +} {/USING AUTOMATIC COVERING INDEX/} + + finish_test From dc41d60aaa9add6f5909cbd3d91c842f365ec773 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Sep 2014 19:51:35 +0000 Subject: [PATCH 03/25] Size reduction and substantial performance increase for cellSizePtr(). FossilOrigin-Name: bc8bbf3207288d160287519c3b7123997996b440 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 25 +++++++++++++++---------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 926eedc817..d9984e2d8c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tune\sthe\squery\splanner\sto\sbe\smore\saggressive\sabout\susing\sautomatic\sindexes\non\sviews\sand\ssubqueries\sfor\swhich\sthere\sis\snot\sopportunity\sto\sdeclare\sa\npersistent\sschema\sindex. -D 2014-09-22T14:30:51.911 +C Size\sreduction\sand\ssubstantial\sperformance\sincrease\sfor\scellSizePtr(). +D 2014-09-22T19:51:35.432 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 6aa61c0e3d20d1d1acc8fb33d8f0ebd675305d3c +F src/btree.c 86cc6efed093b80360489acac4d2daf064a1ad58 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 10a6e510497b471d67ac3dfb19ff256a7d18adf4 -R f6ce79b0bd0352edf4a2c65db2a400a7 +P 41de1643bfc9ae25e20790d707e2789b665baa2b +R 20814d8e71474a3ca52334b7cc5e7228 U drh -Z b1246ec5b6570c51d45a58350d46840c +Z 93d80b8ba28615b25f3f6e1541988990 diff --git a/manifest.uuid b/manifest.uuid index a03afa027d..63137ebd34 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -41de1643bfc9ae25e20790d707e2789b665baa2b \ No newline at end of file +bc8bbf3207288d160287519c3b7123997996b440 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 522e945ac2..46eb5be303 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1057,6 +1057,7 @@ static void btreeParseCell( */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ u8 *pIter = &pCell[pPage->childPtrSize]; + u8 *pEnd; u32 nSize; #ifdef SQLITE_DEBUG @@ -1068,21 +1069,25 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ btreeParseCellPtr(pPage, pCell, &debuginfo); #endif - if( pPage->intKey ){ - u8 *pEnd; - if( pPage->hasData ){ - pIter += getVarint32(pIter, nSize); - }else{ - nSize = 0; + if( pPage->intKey==0 || pPage->hasData ){ + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[9]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIter<&pCell[6] ); } - + pIter++; + }else{ + nSize = 0; + } + if( pPage->intKey ){ /* pIter now points at the 64-bit integer key value, a variable length ** integer. The following block moves pIter to point at the first byte ** past the end of the key value. */ pEnd = &pIter[9]; while( (*pIter++)&0x80 && pItermaxLocal ); @@ -1104,7 +1109,7 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ nSize = 4; } - assert( nSize==debuginfo.nSize ); + assert( nSize==debuginfo.nSize || CORRUPT_DB ); return (u16)nSize; } From a4bb5b0726772476cdf36b4a0182e27a73fab557 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Sep 2014 20:38:10 +0000 Subject: [PATCH 04/25] Fix to payload size overflow detection in the cellSizePtr() change of the previous check-in. FossilOrigin-Name: 7609744014c6a84a8379794a0351a2e9626ec86b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d9984e2d8c..625787bf08 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Size\sreduction\sand\ssubstantial\sperformance\sincrease\sfor\scellSizePtr(). -D 2014-09-22T19:51:35.432 +C Fix\sto\spayload\ssize\soverflow\sdetection\sin\sthe\scellSizePtr()\schange\sof\nthe\sprevious\scheck-in. +D 2014-09-22T20:38:10.316 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 86cc6efed093b80360489acac4d2daf064a1ad58 +F src/btree.c c2645014c525c0b4a8327971c331f55b8747b443 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 41de1643bfc9ae25e20790d707e2789b665baa2b -R 20814d8e71474a3ca52334b7cc5e7228 +P bc8bbf3207288d160287519c3b7123997996b440 +R 5a3408a8ae29b6451a1037ab47048e78 U drh -Z 93d80b8ba28615b25f3f6e1541988990 +Z 93e96edbb124ae02212644cc0f189540 diff --git a/manifest.uuid b/manifest.uuid index 63137ebd34..b44f538e96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bc8bbf3207288d160287519c3b7123997996b440 \ No newline at end of file +7609744014c6a84a8379794a0351a2e9626ec86b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 46eb5be303..1856ecb1a5 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1076,7 +1076,7 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ nSize &= 0x7f; do{ nSize = (nSize<<7) | (*++pIter & 0x7f); - }while( *(pIter)>=0x80 && pIter<&pCell[6] ); + }while( *(pIter)>=0x80 && pIter Date: Tue, 23 Sep 2014 01:40:59 +0000 Subject: [PATCH 05/25] Adjust skip-scan cost estimates slightly so that a full table scan is preferred over a skip-scan to a column with only two distinct values. FossilOrigin-Name: ae9a42b268ad3f7d21a5813bb931e795c6917014 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 5 ++++- test/skipscan1.test | 28 +++++++++++++++++++++++++++- test/skipscan5.test | 6 +----- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 625787bf08..aeea9bdb33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sto\spayload\ssize\soverflow\sdetection\sin\sthe\scellSizePtr()\schange\sof\nthe\sprevious\scheck-in. -D 2014-09-22T20:38:10.316 +C Adjust\sskip-scan\scost\sestimates\sslightly\sso\sthat\sa\sfull\stable\sscan\sis\npreferred\sover\sa\sskip-scan\sto\sa\scolumn\swith\sonly\stwo\sdistinct\svalues. +D 2014-09-23T01:40:59.122 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 3f859ecfada8643ce8255f20d481b5a3e0921662 +F src/where.c a14d3d8042adeb51f81731c1b47b3e481d1cc23a F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -836,10 +836,10 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test 28c7faa41a0d7265040ecb0a0abd90c0904270b2 +F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 -F test/skipscan5.test d8b9692b702745a0e41c23f9da6beac81df01196 +F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 15e1d3014abc3f6d4357ed81b93b82117aefd235 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bc8bbf3207288d160287519c3b7123997996b440 -R 5a3408a8ae29b6451a1037ab47048e78 +P 7609744014c6a84a8379794a0351a2e9626ec86b +R cb94e4a1db7c56387373fb77d2698ee3 U drh -Z 93e96edbb124ae02212644cc0f189540 +Z a13a3c1c4006b80e078cd097de2cdb1a diff --git a/manifest.uuid b/manifest.uuid index b44f538e96..c811540af6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7609744014c6a84a8379794a0351a2e9626ec86b \ No newline at end of file +ae9a42b268ad3f7d21a5813bb931e795c6917014 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d1862b499a..40f95acc49 100644 --- a/src/where.c +++ b/src/where.c @@ -4359,11 +4359,14 @@ static int whereLoopAddBtreeIndex( nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; if( pTerm ){ /* TUNING: When estimating skip-scan for a term that is also indexable, - ** increase the cost of the skip-scan by 2x, to make it a little less + ** multiply the cost of the skip-scan by 2.0, to make it a little less ** desirable than the regular index lookup. */ nIter += 10; assert( 10==sqlite3LogEst(2) ); } pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); pNew->nOut = saved_nOut; pNew->u.btree.nEq = saved_nEq; diff --git a/test/skipscan1.test b/test/skipscan1.test index 8150b012f6..6b9f1209a5 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -245,6 +245,32 @@ do_execsql_test skipscan1-5.3 { SELECT xh, loc FROM t5 WHERE loc >= 'M' AND loc < 'N'; } {/.*COVERING INDEX t5i1 .*/} - +# The column used by the skip-scan needs to be sufficiently selective. +# See the private email from Adi Zaimi to drh@sqlite.org on 2014-09-22. +# +db close +forcedelete test.db +sqlite3 db test.db +do_execsql_test skipscan1-6.1 { + CREATE TABLE t1(a,b,c,d,e,f,g,h varchar(300)); + CREATE INDEX t1ab ON t1(a,b); + ANALYZE sqlite_master; + -- Only two distinct values for the skip-scan column. Skip-scan is not used. + INSERT INTO sqlite_stat1 VALUES('t1','t1ab','500000 250000 125000'); + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} +do_execsql_test skipscan1-6.2 { + -- Four distinct values for the skip-scan column. Skip-scan is used. + UPDATE sqlite_stat1 SET stat='500000 250000 62500'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {/ANY.a. AND b=/} +do_execsql_test skipscan1-6.3 { + -- Two distinct values for the skip-scan column again. Skip-scan is not used. + UPDATE sqlite_stat1 SET stat='500000 125000 62500'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} finish_test diff --git a/test/skipscan5.test b/test/skipscan5.test index 5d6d392998..7c3b166a8c 100644 --- a/test/skipscan5.test +++ b/test/skipscan5.test @@ -108,7 +108,7 @@ foreach {tn dbenc coll} { 3 { c > 'q' } {/*ANY(a) AND ANY(b) AND c>?*/} 4 { c > 'e' } {/*SCAN TABLE t2*/} 5 { c < 'q' } {/*SCAN TABLE t2*/} - 4 { c < 'e' } {/*ANY(a) AND ANY(b) AND c Date: Tue, 23 Sep 2014 18:30:00 +0000 Subject: [PATCH 06/25] Add the "multiplex_truncate" PRAGMA to the multiplexor extension, for querying and setting the truncate flag on a database connection. FossilOrigin-Name: d2962a5f388f30a02429e0c8b87399f482b5604c --- manifest | 13 ++--- manifest.uuid | 2 +- src/test_multiplex.c | 20 ++++++++ test/multiplex4.test | 114 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 test/multiplex4.test diff --git a/manifest b/manifest index aeea9bdb33..7873e80eab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sskip-scan\scost\sestimates\sslightly\sso\sthat\sa\sfull\stable\sscan\sis\npreferred\sover\sa\sskip-scan\sto\sa\scolumn\swith\sonly\stwo\sdistinct\svalues. -D 2014-09-23T01:40:59.122 +C Add\sthe\s"multiplex_truncate"\sPRAGMA\sto\sthe\smultiplexor\sextension,\sfor\nquerying\sand\ssetting\sthe\struncate\sflag\son\sa\sdatabase\sconnection. +D 2014-09-23T18:30:00.961 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -262,7 +262,7 @@ F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 F src/test_malloc.c ba34143f941a9d74b30bbffc8818389bb73a1ca2 -F src/test_multiplex.c ca90057438b63bf0840ebb84d0ef050624519a76 +F src/test_multiplex.c caadb62cc777268b4f8fb94d5b27b80156c8f7c0 F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f F src/test_onefile.c 0396f220561f3b4eedc450cef26d40c593c69a25 @@ -729,6 +729,7 @@ F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/multiplex.test efd015ca0b5b4a57dc9535b8feb1273eebeadb60 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 +F test/multiplex4.test d3e8a5a522c51cbf3ed1c5b0bd496be02c29d7b1 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a @@ -1199,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7609744014c6a84a8379794a0351a2e9626ec86b -R cb94e4a1db7c56387373fb77d2698ee3 +P ae9a42b268ad3f7d21a5813bb931e795c6917014 +R 38a39c43ba2ffbcd445b1883953d7c57 U drh -Z a13a3c1c4006b80e078cd097de2cdb1a +Z 9797dd2e522d2cafa5364dec0ef721e8 diff --git a/manifest.uuid b/manifest.uuid index c811540af6..a1ae8eed66 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae9a42b268ad3f7d21a5813bb931e795c6917014 \ No newline at end of file +d2962a5f388f30a02429e0c8b87399f482b5604c \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 427cc65ad7..99819371ce 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -1002,6 +1002,26 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ /* no-op these */ rc = SQLITE_OK; break; + case SQLITE_FCNTL_PRAGMA: { + char **aFcntl = (char**)pArg; + if( aFcntl[1] && sqlite3_stricmp(aFcntl[1],"multiplex_truncate")==0 ){ + if( aFcntl[2] && aFcntl[2][0] ){ + if( sqlite3_stricmp(aFcntl[2], "on")==0 + || sqlite3_stricmp(aFcntl[2], "1")==0 ){ + pGroup->bTruncate = 1; + }else + if( sqlite3_stricmp(aFcntl[2], "off")==0 + || sqlite3_stricmp(aFcntl[2], "0")==0 ){ + pGroup->bTruncate = 0; + } + } + aFcntl[0] = sqlite3_mprintf(pGroup->bTruncate ? "on" : "off"); + rc = SQLITE_OK; + break; + } + /* If the multiplexor does not handle the pragma, pass it through + ** into the default case. */ + } default: pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ diff --git a/test/multiplex4.test b/test/multiplex4.test new file mode 100644 index 0000000000..9c304c314d --- /dev/null +++ b/test/multiplex4.test @@ -0,0 +1,114 @@ +# 2014-09-25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the "truncate" option in the multiplexor. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix multiplex4 + +db close +sqlite3_shutdown +sqlite3_multiplex_initialize {} 0 + +# delete all filesl with the base name of $basename +# +proc multiplex_delete_db {basename} { + foreach file [glob -nocomplain $basename.*] { + forcedelete $file + } +} + +# Return a sorted list of all files with the base name of $basename. +# Except, delete all text from the end of $basename through the NNN +# suffix on the end of the filename. +# +proc multiplex_file_list {basename} { + set x {} + foreach file [glob -nocomplain $basename.*] { + regsub "^$basename\\..*(\\d\\d\\d)\$" $file $basename.\\1 file + lappend x $file + } + return [lsort $x] +} + +do_test multiplex4-1.0 { + multiplex_delete_db mx4test + sqlite3 db {file:mx4test.db?chunksize=10&truncate=1} -uri 1 -vfs multiplex + db eval { + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES(randomblob(250000)); + } + multiplex_file_list mx4test +} {mx4test.001 mx4test.db} + +do_test multiplex4-1.1 { + db eval { + DELETE FROM t1; + VACUUM; + } + multiplex_file_list mx4test +} {mx4test.db} + +do_test multiplex4-1.2 { + db eval {PRAGMA multiplex_truncate} +} {on} +do_test multiplex4-1.3 { + db eval {PRAGMA multiplex_truncate=off} +} {off} +do_test multiplex4-1.4 { + db eval {PRAGMA multiplex_truncate} +} {off} +do_test multiplex4-1.5 { + db eval {PRAGMA multiplex_truncate=on} +} {on} +do_test multiplex4-1.6 { + db eval {PRAGMA multiplex_truncate} +} {on} +do_test multiplex4-1.7 { + db eval {PRAGMA multiplex_truncate=0} +} {off} +do_test multiplex4-1.8 { + db eval {PRAGMA multiplex_truncate=1} +} {on} +do_test multiplex4-1.9 { + db eval {PRAGMA multiplex_truncate=0} +} {off} + +do_test multiplex4-1.10 { + db eval { + INSERT INTO t1(x) VALUES(randomblob(250000)); + } + multiplex_file_list mx4test +} {mx4test.001 mx4test.db} + +do_test multiplex4-1.11 { + db eval { + DELETE FROM t1; + VACUUM; + } + multiplex_file_list mx4test +} {mx4test.001 mx4test.db} + +do_test multiplex4-1.12 { + db eval { + PRAGMA multiplex_truncate=ON; + DROP TABLE t1; + VACUUM; + } + multiplex_file_list mx4test +} {mx4test.db} + +catch { db close } +forcedelete mx4test.db +sqlite3_multiplex_shutdown +finish_test From ab1cc58b7018a2bcfd9a3b2107bd2bbb6cbbf0f8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Sep 2014 21:25:19 +0000 Subject: [PATCH 07/25] Simplify the CellInfo structure for a size reduction and performance improvement. FossilOrigin-Name: bf59df66b3613c38cfb13a68091b8328ebb22c78 --- manifest | 14 ++++++------ manifest.uuid | 2 +- src/btree.c | 60 ++++++++++++++++++++++---------------------------- src/btreeInt.h | 10 ++++----- 4 files changed, 38 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index 7873e80eab..70db7cda92 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"multiplex_truncate"\sPRAGMA\sto\sthe\smultiplexor\sextension,\sfor\nquerying\sand\ssetting\sthe\struncate\sflag\son\sa\sdatabase\sconnection. -D 2014-09-23T18:30:00.961 +C Simplify\sthe\sCellInfo\sstructure\sfor\sa\ssize\sreduction\sand\sperformance\nimprovement. +D 2014-09-23T21:25:19.710 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,9 +172,9 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c c2645014c525c0b4a8327971c331f55b8747b443 +F src/btree.c 2a7c67d474624732612f97a89e34cf85f8cd4905 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 -F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc +F src/btreeInt.h a5a869ec2c3e56ee9e214ee748d7942716be0340 F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ae9a42b268ad3f7d21a5813bb931e795c6917014 -R 38a39c43ba2ffbcd445b1883953d7c57 +P d2962a5f388f30a02429e0c8b87399f482b5604c +R 3633e95932853227476c5d6e8f02e193 U drh -Z 9797dd2e522d2cafa5364dec0ef721e8 +Z 6131f39965fefa27be7e69636d8311d7 diff --git a/manifest.uuid b/manifest.uuid index a1ae8eed66..24ab1f920d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2962a5f388f30a02429e0c8b87399f482b5604c \ No newline at end of file +bf59df66b3613c38cfb13a68091b8328ebb22c78 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 1856ecb1a5..233f674f0f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -977,38 +977,33 @@ static void btreeParseCellPtr( u8 *pCell, /* Pointer to the cell text. */ CellInfo *pInfo /* Fill in this structure */ ){ - u16 n; /* Number bytes in cell content header */ + u8 *pIter = &pCell[pPage->childPtrSize]; u32 nPayload; /* Number of bytes of cell payload */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - - pInfo->pCell = pCell; assert( pPage->leaf==0 || pPage->leaf==1 ); - n = pPage->childPtrSize; - assert( n==4-4*pPage->leaf ); if( pPage->intKey ){ if( pPage->hasData ){ - assert( n==0 ); - n = getVarint32(pCell, nPayload); + assert( pIter==pCell ); + pIter += getVarint32(pIter, nPayload); }else{ nPayload = 0; } - n += getVarint(&pCell[n], (u64*)&pInfo->nKey); - pInfo->nData = nPayload; + pIter += getVarint(pIter, (u64*)&pInfo->nKey); }else{ - pInfo->nData = 0; - n += getVarint32(&pCell[n], nPayload); + pIter += getVarint32(pIter, nPayload); pInfo->nKey = nPayload; } pInfo->nPayload = nPayload; - pInfo->nHeader = n; + pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==pPage->maxLocal+1 ); - if( likely(nPayload<=pPage->maxLocal) ){ + if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ - if( (pInfo->nSize = (u16)(n+nPayload))<4 ) pInfo->nSize = 4; + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; pInfo->iOverflow = 0; }else{ @@ -1035,7 +1030,7 @@ static void btreeParseCellPtr( }else{ pInfo->nLocal = (u16)minLocal; } - pInfo->iOverflow = (u16)(pInfo->nLocal + n); + pInfo->iOverflow = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell); pInfo->nSize = pInfo->iOverflow + 4; } } @@ -1132,7 +1127,6 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ if( *pRC ) return; assert( pCell!=0 ); btreeParseCellPtr(pPage, pCell, &info); - assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload ); if( info.iOverflow ){ Pgno ovfl = get4byte(&pCell[info.iOverflow]); ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); @@ -3866,8 +3860,9 @@ int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); + assert( pCur->apPage[pCur->iPage]->intKey==1 ); getCellInfo(pCur); - *pSize = pCur->info.nData; + *pSize = pCur->info.nPayload; return SQLITE_OK; } @@ -4018,7 +4013,6 @@ static int accessPayload( ){ unsigned char *aPayload; int rc = SQLITE_OK; - u32 nKey; int iIdx = 0; MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ @@ -4033,15 +4027,13 @@ static int accessPayload( assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); - aPayload = pCur->info.pCell + pCur->info.nHeader; - nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); + aPayload = pCur->info.pPayload; #ifdef SQLITE_DIRECT_OVERFLOW_READ - bEnd = (offset+amt==nKey+pCur->info.nData); + bEnd = offset+amt==pCur->info.nPayload; #endif + assert( offset+amt <= pCur->info.nPayload ); - if( NEVER(offset+amt > nKey+pCur->info.nData) - || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] - ){ + if( &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ){ /* Trying to read or write past the end of the data is an error */ return SQLITE_CORRUPT_BKPT; } @@ -4275,7 +4267,7 @@ static const void *fetchPayload( assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); assert( pCur->info.nSize>0 ); *pAmt = pCur->info.nLocal; - return (void*)(pCur->info.pCell + pCur->info.nHeader); + return (void*)pCur->info.pPayload; } @@ -5640,9 +5632,10 @@ static int fillInCell( } nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); btreeParseCellPtr(pPage, pCell, &info); - assert( info.nHeader==nHeader ); + assert( nHeader=(int)(info.pPayload - pCell) ); assert( info.nKey==nKey ); - assert( info.nData==(u32)(nData+nZero) ); + assert( pPage->intKey==0 || info.nPayload==(u32)(nData+nZero) ); + assert( pPage->intKey==1 || info.nPayload==nKey ); /* Fill in the payload */ nPayload = nData + nZero; @@ -8086,19 +8079,18 @@ static int checkTreePage( "On tree page %d cell %d: ", iPage, i); pCell = findCell(pPage,i); btreeParseCellPtr(pPage, pCell, &info); - sz = info.nData; - if( !pPage->intKey ) sz += (int)info.nKey; + sz = info.nPayload; /* For intKey pages, check that the keys are in order. */ - else if( i==0 ) nMinKey = nMaxKey = info.nKey; - else{ - if( info.nKey <= nMaxKey ){ + if( pPage->intKey ){ + if( i==0 ){ + nMinKey = nMaxKey = info.nKey; + }else if( info.nKey <= nMaxKey ){ checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); + "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); } nMaxKey = info.nKey; } - assert( sz==info.nPayload ); if( (sz>info.nLocal) && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) ){ diff --git a/src/btreeInt.h b/src/btreeInt.h index fbfe47f6bc..c9ff79c90f 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -456,12 +456,10 @@ struct BtShared { */ typedef struct CellInfo CellInfo; struct CellInfo { - i64 nKey; /* The key for INTKEY tables, or number of bytes in key */ - u8 *pCell; /* Pointer to the start of cell content */ - u32 nData; /* Number of bytes of data */ - u32 nPayload; /* Total amount of payload */ - u16 nHeader; /* Size of the cell content header in bytes */ - u16 nLocal; /* Amount of payload held locally */ + i64 nKey; /* The key for INTKEY tables, or nPayload otherwise */ + u8 *pPayload; /* Pointer to the start of payload */ + u32 nPayload; /* Bytes of payload */ + u16 nLocal; /* Amount of payload held locally, not on overflow */ u16 iOverflow; /* Offset to overflow page number. Zero if no overflow */ u16 nSize; /* Size of the cell content on the main b-tree page */ }; From 6200c88123e5933dc81ec132d0c03f8bff86a07b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Sep 2014 22:36:25 +0000 Subject: [PATCH 08/25] Avoid calling btreeParseCellPtr() from within fillInCell() since most of what btreeParseCellPtr() computes is ignored by fillInCell(). Instead, have fillInCell() compute the values it needs inline. Performance improvement. FossilOrigin-Name: 4147f6671e3faa8ddffab8387a6c7d9b5b962fc8 --- manifest | 12 ++++----- manifest.uuid | 2 +- src/btree.c | 69 +++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 58 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 70db7cda92..2923bfbe6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\sCellInfo\sstructure\sfor\sa\ssize\sreduction\sand\sperformance\nimprovement. -D 2014-09-23T21:25:19.710 +C Avoid\scalling\sbtreeParseCellPtr()\sfrom\swithin\sfillInCell()\ssince\smost\sof\nwhat\sbtreeParseCellPtr()\scomputes\sis\signored\sby\sfillInCell().\s\sInstead,\shave\nfillInCell()\scompute\sthe\svalues\sit\sneeds\sinline.\s\sPerformance\simprovement. +D 2014-09-23T22:36:25.858 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 2a7c67d474624732612f97a89e34cf85f8cd4905 +F src/btree.c 11cf36074a9829bad4506e8486bfd2431168bb54 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h a5a869ec2c3e56ee9e214ee748d7942716be0340 F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d2962a5f388f30a02429e0c8b87399f482b5604c -R 3633e95932853227476c5d6e8f02e193 +P bf59df66b3613c38cfb13a68091b8328ebb22c78 +R 7c25e7d21782a499afecd58eb6881944 U drh -Z 6131f39965fefa27be7e69636d8311d7 +Z 9128d4e264c0fc76a3ba82d24c937cbe diff --git a/manifest.uuid b/manifest.uuid index 24ab1f920d..9320755605 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf59df66b3613c38cfb13a68091b8328ebb22c78 \ No newline at end of file +4147f6671e3faa8ddffab8387a6c7d9b5b962fc8 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 233f674f0f..5d05e98d85 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5611,7 +5611,6 @@ static int fillInCell( BtShared *pBt = pPage->pBt; Pgno pgnoOvfl = 0; int nHeader; - CellInfo info; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); @@ -5621,24 +5620,18 @@ static int fillInCell( || sqlite3PagerIswriteable(pPage->pDbPage) ); /* Fill in the header. */ - nHeader = 0; - if( !pPage->leaf ){ - nHeader += 4; - } + nHeader = pPage->childPtrSize; + nPayload = nData + nZero; if( pPage->hasData ){ - nHeader += putVarint32(&pCell[nHeader], nData+nZero); + assert( pPage->intKey ); + nHeader += putVarint32(&pCell[nHeader], nPayload); }else{ - nData = nZero = 0; + assert( nData==0 ); + assert( nZero==0 ); } nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); - btreeParseCellPtr(pPage, pCell, &info); - assert( nHeader=(int)(info.pPayload - pCell) ); - assert( info.nKey==nKey ); - assert( pPage->intKey==0 || info.nPayload==(u32)(nData+nZero) ); - assert( pPage->intKey==1 || info.nPayload==nKey ); - /* Fill in the payload */ - nPayload = nData + nZero; + /* Fill in the payload size */ if( pPage->intKey ){ pSrc = pData; nSrc = nData; @@ -5647,15 +5640,55 @@ static int fillInCell( if( NEVER(nKey>0x7fffffff || pKey==0) ){ return SQLITE_CORRUPT_BKPT; } - nPayload += (int)nKey; + nPayload = (int)nKey; pSrc = pKey; nSrc = (int)nKey; } - *pnSize = info.nSize; - spaceLeft = info.nLocal; + if( nPayload<=pPage->maxLocal ){ + n = nHeader + nPayload; + testcase( n==3 ); + testcase( n==4 ); + if( n<4 ) n = 4; + *pnSize = n; + spaceLeft = nPayload; + pPrior = pCell; + }else{ + int mn = pPage->minLocal; + n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4); + testcase( n==pPage->maxLocal ); + testcase( n==pPage->maxLocal+1 ); + if( n > pPage->maxLocal ) n = mn; + spaceLeft = n; + *pnSize = n + nHeader + 4; + pPrior = &pCell[nHeader+n]; + } pPayload = &pCell[nHeader]; - pPrior = &pCell[info.iOverflow]; + /* At this point variables should be set as follows: + ** + ** nPayload Total payload size in bytes + ** pPayload Begin writing payload here + ** spaceLeft Space available at pPayload. If nPayload>spaceLeft, + ** that means content must spill into overflow pages. + ** *pnSize Size of the local cell (not counting overflow pages) + ** pPrior Where to write the pgno of the first overflow page + ** + ** Use a call to btreeParseCellPtr() to verify that the values above + ** were computed correctly. + */ +#if SQLITE_DEBUG + { + CellInfo info; + btreeParseCellPtr(pPage, pCell, &info); + assert( nHeader=(int)(info.pPayload - pCell) ); + assert( info.nKey==nKey ); + assert( *pnSize == info.nSize ); + assert( spaceLeft == info.nLocal ); + assert( pPrior == &pCell[info.iOverflow] ); + } +#endif + + /* Write the payload into the local Cell and any extra into overflow pages */ while( nPayload>0 ){ if( spaceLeft==0 ){ #ifndef SQLITE_OMIT_AUTOVACUUM From c46838309c525b1b441f12f42b1cb493cdca89bc Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Sep 2014 23:12:53 +0000 Subject: [PATCH 09/25] Remove an unused C-preprocessor macro. No functional changes to the code. FossilOrigin-Name: f480582ccae0e9a917d4523191025bd16016ba64 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 7 +------ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 2923bfbe6d..b80dfe9ce8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\sbtreeParseCellPtr()\sfrom\swithin\sfillInCell()\ssince\smost\sof\nwhat\sbtreeParseCellPtr()\scomputes\sis\signored\sby\sfillInCell().\s\sInstead,\shave\nfillInCell()\scompute\sthe\svalues\sit\sneeds\sinline.\s\sPerformance\simprovement. -D 2014-09-23T22:36:25.858 +C Remove\san\sunused\sC-preprocessor\smacro.\s\sNo\sfunctional\schanges\sto\sthe\scode. +D 2014-09-23T23:12:53.118 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 11cf36074a9829bad4506e8486bfd2431168bb54 +F src/btree.c b10d4c349e62faca4ff86f168a02ca9988d1fd6f F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h a5a869ec2c3e56ee9e214ee748d7942716be0340 F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bf59df66b3613c38cfb13a68091b8328ebb22c78 -R 7c25e7d21782a499afecd58eb6881944 +P 4147f6671e3faa8ddffab8387a6c7d9b5b962fc8 +R ff3738f15db60d92a05b3981ec1f9ca5 U drh -Z 9128d4e264c0fc76a3ba82d24c937cbe +Z 53dcb91c2d5e4a116ccba455c707085a diff --git a/manifest.uuid b/manifest.uuid index 9320755605..71bf446c8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4147f6671e3faa8ddffab8387a6c7d9b5b962fc8 \ No newline at end of file +f480582ccae0e9a917d4523191025bd16016ba64 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5d05e98d85..5a0fd4e184 100644 --- a/src/btree.c +++ b/src/btree.c @@ -968,9 +968,6 @@ static u8 *findOverflowCell(MemPage *pPage, int iCell){ ** are two versions of this function. btreeParseCell() takes a ** cell index as the second argument and btreeParseCellPtr() ** takes a pointer to the body of the cell as its second argument. -** -** Within this file, the parseCell() macro can be called instead of -** btreeParseCellPtr(). Using some compilers, this will be faster. */ static void btreeParseCellPtr( MemPage *pPage, /* Page containing the cell */ @@ -1034,14 +1031,12 @@ static void btreeParseCellPtr( pInfo->nSize = pInfo->iOverflow + 4; } } -#define parseCell(pPage, iCell, pInfo) \ - btreeParseCellPtr((pPage), findCell((pPage), (iCell)), (pInfo)) static void btreeParseCell( MemPage *pPage, /* Page containing the cell */ int iCell, /* The cell index. First cell is 0 */ CellInfo *pInfo /* Fill in this structure */ ){ - parseCell(pPage, iCell, pInfo); + btreeParseCellPtr(pPage, findCell(pPage, iCell), pInfo); } /* From 3e28ff5cb53f51f2903d5ec056a71040bdddf5de Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 00:59:08 +0000 Subject: [PATCH 10/25] Add the MemPage.noPayload boolean and use it to help cellSizePtr() and btreeParseCellPtr() run faster. FossilOrigin-Name: 8e3375313ebbf26b68561f3ed31d2a488222e5d0 --- manifest | 14 ++++----- manifest.uuid | 2 +- src/btree.c | 85 ++++++++++++++++++++++++++------------------------ src/btreeInt.h | 7 +++-- 4 files changed, 56 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index b80dfe9ce8..41868133a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\sC-preprocessor\smacro.\s\sNo\sfunctional\schanges\sto\sthe\scode. -D 2014-09-23T23:12:53.118 +C Add\sthe\sMemPage.noPayload\sboolean\sand\suse\sit\sto\shelp\ncellSizePtr()\sand\sbtreeParseCellPtr()\srun\sfaster. +D 2014-09-24T00:59:08.082 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,9 +172,9 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c b10d4c349e62faca4ff86f168a02ca9988d1fd6f +F src/btree.c d64b3c5569bda0e7fbe9bc9388e17eaf70d63dec F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 -F src/btreeInt.h a5a869ec2c3e56ee9e214ee748d7942716be0340 +F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4147f6671e3faa8ddffab8387a6c7d9b5b962fc8 -R ff3738f15db60d92a05b3981ec1f9ca5 +P f480582ccae0e9a917d4523191025bd16016ba64 +R a8f50a441085b36b058ddd2aec76446f U drh -Z 53dcb91c2d5e4a116ccba455c707085a +Z b0d06d7ed20970cccde4605a2a98dc92 diff --git a/manifest.uuid b/manifest.uuid index 71bf446c8d..dae8a6ddc0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f480582ccae0e9a917d4523191025bd16016ba64 \ No newline at end of file +8e3375313ebbf26b68561f3ed31d2a488222e5d0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5a0fd4e184..8644538abb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -974,20 +974,25 @@ static void btreeParseCellPtr( u8 *pCell, /* Pointer to the cell text. */ CellInfo *pInfo /* Fill in this structure */ ){ - u8 *pIter = &pCell[pPage->childPtrSize]; + u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); - if( pPage->intKey ){ - if( pPage->hasData ){ - assert( pIter==pCell ); - pIter += getVarint32(pIter, nPayload); - }else{ - nPayload = 0; - } + if( pPage->intKeyLeaf ){ + assert( pPage->childPtrSize==0 ); + pIter = pCell + getVarint32(pCell, nPayload); pIter += getVarint(pIter, (u64*)&pInfo->nKey); + }else if( pPage->noPayload ){ + assert( pPage->childPtrSize==4 ); + pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); + pInfo->nPayload = 0; + pInfo->nLocal = 0; + pInfo->iOverflow = 0; + pInfo->pPayload = 0; + return; }else{ + pIter = pCell + pPage->childPtrSize; pIter += getVarint32(pIter, nPayload); pInfo->nKey = nPayload; } @@ -1046,9 +1051,9 @@ static void btreeParseCell( ** the space used by the cell pointer. */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = &pCell[pPage->childPtrSize]; - u8 *pEnd; - u32 nSize; + u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ #ifdef SQLITE_DEBUG /* The value returned by this function should always be the same as @@ -1059,19 +1064,21 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ btreeParseCellPtr(pPage, pCell, &debuginfo); #endif - if( pPage->intKey==0 || pPage->hasData ){ - nSize = *pIter; - if( nSize>=0x80 ){ - pEnd = &pIter[9]; - nSize &= 0x7f; - do{ - nSize = (nSize<<7) | (*++pIter & 0x7f); - }while( *(pIter)>=0x80 && pIternoPayload ){ + pEnd = &pIter[9]; + while( (*pIter++)&0x80 && pIterchildPtrSize==4 ); + return (u16)(pIter - pCell); } + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[9]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIterintKey ){ /* pIter now points at the 64-bit integer key value, a variable length ** integer. The following block moves pIter to point at the first byte @@ -1079,10 +1086,12 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ pEnd = &pIter[9]; while( (*pIter++)&0x80 && pItermaxLocal ); testcase( nSize==pPage->maxLocal+1 ); - if( nSize>pPage->maxLocal ){ + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + if( nSize<4 ) nSize = 4; + }else{ int minLocal = pPage->minLocal; nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); testcase( nSize==pPage->maxLocal ); @@ -1090,15 +1099,8 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ if( nSize>pPage->maxLocal ){ nSize = minLocal; } - nSize += 4; + nSize += 4 + (u16)(pIter - pCell); } - nSize += (u32)(pIter - pCell); - - /* The minimum size of any cell is 4 bytes. */ - if( nSize<4 ){ - nSize = 4; - } - assert( nSize==debuginfo.nSize || CORRUPT_DB ); return (u16)nSize; } @@ -1442,12 +1444,14 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ pPage->intKey = 1; - pPage->hasData = pPage->leaf; + pPage->intKeyLeaf = pPage->leaf; + pPage->noPayload = !pPage->leaf; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ pPage->intKey = 0; - pPage->hasData = 0; + pPage->intKeyLeaf = 0; + pPage->noPayload = 0; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ @@ -3855,7 +3859,7 @@ int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - assert( pCur->apPage[pCur->iPage]->intKey==1 ); + assert( pCur->apPage[pCur->iPage]->intKeyLeaf==1 ); getCellInfo(pCur); *pSize = pCur->info.nPayload; return SQLITE_OK; @@ -4690,7 +4694,7 @@ int sqlite3BtreeMovetoUnpacked( for(;;){ i64 nCellKey; pCell = findCell(pPage, idx) + pPage->childPtrSize; - if( pPage->hasData ){ + if( pPage->intKeyLeaf ){ while( 0x80 <= *(pCell++) ){ if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; } @@ -5617,8 +5621,7 @@ static int fillInCell( /* Fill in the header. */ nHeader = pPage->childPtrSize; nPayload = nData + nZero; - if( pPage->hasData ){ - assert( pPage->intKey ); + if( pPage->intKeyLeaf ){ nHeader += putVarint32(&pCell[nHeader], nPayload); }else{ assert( nData==0 ); @@ -6391,7 +6394,7 @@ static int balance_nonroot( ** leafData: 1 if pPage holds key+data and pParent holds only keys. */ leafCorrection = apOld[0]->leaf*4; - leafData = apOld[0]->hasData; + leafData = apOld[0]->intKeyLeaf; for(i=0; ipDbPage); if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_QUICKBALANCE - if( pPage->hasData + if( pPage->intKeyLeaf && pPage->nOverflow==1 && pPage->aiOvfl[0]==pPage->nCell && pParent->pgno!=1 diff --git a/src/btreeInt.h b/src/btreeInt.h index c9ff79c90f..df9684e8e9 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -273,9 +273,10 @@ typedef struct BtLock BtLock; struct MemPage { u8 isInit; /* True if previously initialized. MUST BE FIRST! */ u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ - u8 intKey; /* True if intkey flag is set */ - u8 leaf; /* True if leaf flag is set */ - u8 hasData; /* True if this page stores data */ + u8 intKey; /* True if table b-trees. False for index b-trees */ + u8 intKeyLeaf; /* True if the leaf of an intKey table */ + u8 noPayload; /* True if internal intKey page (thus w/o data) */ + u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ u8 max1bytePayload; /* min(maxLocal,127) */ From 3f3874030cd8cf62f026ba46fe9c5abbef0d8a44 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 01:23:00 +0000 Subject: [PATCH 11/25] Shorten all lines of source code in btree.c to at most 80 characters. No logical changes. FossilOrigin-Name: 5dd41cdbfebdd088ebd9a90a394ee296c207ad90 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 54 ++++++++++++++++++++++++++++----------------------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 41868133a8..a69f68fbf0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sMemPage.noPayload\sboolean\sand\suse\sit\sto\shelp\ncellSizePtr()\sand\sbtreeParseCellPtr()\srun\sfaster. -D 2014-09-24T00:59:08.082 +C Shorten\sall\slines\sof\ssource\scode\sin\sbtree.c\sto\sat\smost\s80\scharacters.\nNo\slogical\schanges. +D 2014-09-24T01:23:00.817 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c d64b3c5569bda0e7fbe9bc9388e17eaf70d63dec +F src/btree.c 3732a278d80867b06f8ed3dfa95338d021f874b0 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f480582ccae0e9a917d4523191025bd16016ba64 -R a8f50a441085b36b058ddd2aec76446f +P 8e3375313ebbf26b68561f3ed31d2a488222e5d0 +R 7f460f85115b9f5ff284729f30ee857e U drh -Z b0d06d7ed20970cccde4605a2a98dc92 +Z 15056c33153d13d2de1255f45fb692fc diff --git a/manifest.uuid b/manifest.uuid index dae8a6ddc0..57ede99673 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e3375313ebbf26b68561f3ed31d2a488222e5d0 \ No newline at end of file +5dd41cdbfebdd088ebd9a90a394ee296c207ad90 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 8644538abb..b943751253 100644 --- a/src/btree.c +++ b/src/btree.c @@ -487,7 +487,9 @@ static void invalidateIncrblobCursors( BtShared *pBt = pBtree->pBt; assert( sqlite3BtreeHoldsMutex(pBtree) ); for(p=pBt->pCursor; p; p=p->pNext){ - if( (p->curFlags & BTCF_Incrblob)!=0 && (isClearTable || p->info.nKey==iRow) ){ + if( (p->curFlags & BTCF_Incrblob)!=0 + && (isClearTable || p->info.nKey==iRow) + ){ p->eState = CURSOR_INVALID; } } @@ -660,9 +662,9 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ ** broken out from its caller to avoid unnecessary stack pointer movement. */ static int SQLITE_NOINLINE saveCursorsOnList( - BtCursor *p, /* The first cursor that needs saving */ - Pgno iRoot, /* Only save cursor with this iRoot. Save all if zero */ - BtCursor *pExcept /* Do not save this cursor */ + BtCursor *p, /* The first cursor that needs saving */ + Pgno iRoot, /* Only save cursor with this iRoot. Save all if zero */ + BtCursor *pExcept /* Do not save this cursor */ ){ do{ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ @@ -1051,9 +1053,9 @@ static void btreeParseCell( ** the space used by the cell pointer. */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ - u8 *pEnd; /* End mark for a varint */ - u32 nSize; /* Size value to return */ + u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ #ifdef SQLITE_DEBUG /* The value returned by this function should always be the same as @@ -1336,7 +1338,7 @@ defragment_page: ** routine and return SQLITE_CORRUPT if any problems are found. */ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ - u16 iPtr; /* Address of pointer to next freeblock */ + u16 iPtr; /* Address of ptr to next freeblock */ u16 iFreeBlk; /* Address of the next freeblock */ u8 hdr; /* Page header size. 0 or 100 */ u8 nFrag = 0; /* Reduction in fragmentation */ @@ -1388,9 +1390,9 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ iFreeBlk = get2byte(&data[iFreeBlk]); } - /* If iPtr is another freeblock (that is, if iPtr is not the freelist pointer - ** in the page header) then check to see if iStart should be coalesced - ** onto the end of iPtr. + /* If iPtr is another freeblock (that is, if iPtr is not the freelist + ** pointer in the page header) then check to see if iStart should be + ** coalesced onto the end of iPtr. */ if( iPtr>hdr+1 ){ int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); @@ -4016,14 +4018,14 @@ static int accessPayload( MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - int bEnd; /* True if reading to end of data */ + int bEnd; /* True if reading to end of data */ #endif assert( pPage ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->aiIdx[pCur->iPage]nCell ); assert( cursorHoldsMutex(pCur) ); - assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ + assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); aPayload = pCur->info.pPayload; @@ -4088,7 +4090,9 @@ static int accessPayload( ** entry for the first required overflow page is valid, skip ** directly to it. */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 && pCur->aOverflow[offset/ovflSize] ){ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 + && pCur->aOverflow[offset/ovflSize] + ){ iIdx = (offset/ovflSize); nextPage = pCur->aOverflow[iIdx]; offset = (offset%ovflSize); @@ -4953,9 +4957,9 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** ** The main entry point is sqlite3BtreePrevious(). That routine is optimized ** for the common case of merely decrementing the cell counter BtCursor.aiIdx -** to the previous cell on the current page. The (slower) btreePrevious() helper -** routine is called when it is necessary to move to a different page or -** to restore the cursor. +** to the previous cell on the current page. The (slower) btreePrevious() +** helper routine is called when it is necessary to move to a different page +** or to restore the cursor. ** ** The calling function will set *pRes to 0 or 1. The initial *pRes value ** will be 1 if the cursor being stepped corresponds to an SQL index and @@ -5283,7 +5287,7 @@ static int allocateBtreePage( memcpy(&aData[8+closest*4], &aData[4+k*4], 4); } put4byte(&aData[4], k-1); - noContent = !btreeGetHasContent(pBt, *pPgno) ? PAGER_GET_NOCONTENT : 0; + noContent = !btreeGetHasContent(pBt, *pPgno)? PAGER_GET_NOCONTENT : 0; rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); @@ -5316,7 +5320,7 @@ static int allocateBtreePage( ** here are confined to those pages that lie between the end of the ** database image and the end of the database file. */ - int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate)) ? PAGER_GET_NOCONTENT : 0; + int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate))? PAGER_GET_NOCONTENT:0; rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc ) return rc; @@ -7089,7 +7093,8 @@ int sqlite3BtreeInsert( } assert( cursorHoldsMutex(pCur) ); - assert( (pCur->curFlags & BTCF_WriteFlag)!=0 && pBt->inTransaction==TRANS_WRITE + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 + && pBt->inTransaction==TRANS_WRITE && (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); @@ -7122,7 +7127,8 @@ int sqlite3BtreeInsert( /* If the cursor is currently on the last row and we are appending a ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto() ** call */ - if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 && pCur->info.nKey==nKey-1 ){ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 + && pCur->info.nKey==nKey-1 ){ loc = -1; } } @@ -8144,7 +8150,7 @@ static int checkTreePage( checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); } #endif - d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0 ? NULL : &nMaxKey); + d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0?NULL:&nMaxKey); if( i>0 && d2!=depth ){ checkAppendMsg(pCheck, zContext, "Child page depth differs"); } @@ -8161,7 +8167,7 @@ static int checkTreePage( checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); } #endif - checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell ? NULL : &nMaxKey); + checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell?NULL:&nMaxKey); } /* For intKey leaf pages, check that the min/max keys are in order @@ -8554,7 +8560,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ ** required in case any of them are holding references to an xFetch ** version of the b-tree page modified by the accessPayload call below. ** - ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** Note that pCsr must be open on a INTKEY table and saveCursorPosition() ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence ** saveAllCursors can only return SQLITE_OK. */ From 9bfdc250622df58dd87e34566ccb23d5910b40a6 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 02:05:41 +0000 Subject: [PATCH 12/25] Have the clearCell() routine return the cell size to the caller, rather than have the caller make a separate call to cellSizePtr(). FossilOrigin-Name: f21d217583c205dc17f98bb4877fd4ed98cefcb1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 22 +++++++++++++++------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a69f68fbf0..209072a802 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Shorten\sall\slines\sof\ssource\scode\sin\sbtree.c\sto\sat\smost\s80\scharacters.\nNo\slogical\schanges. -D 2014-09-24T01:23:00.817 +C Have\sthe\sclearCell()\sroutine\sreturn\sthe\scell\ssize\sto\sthe\scaller,\srather\nthan\shave\sthe\scaller\smake\sa\sseparate\scall\sto\scellSizePtr(). +D 2014-09-24T02:05:41.968 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 3732a278d80867b06f8ed3dfa95338d021f874b0 +F src/btree.c 183d62b37358f95d2ffac796f8491591a3456362 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8e3375313ebbf26b68561f3ed31d2a488222e5d0 -R 7f460f85115b9f5ff284729f30ee857e +P 5dd41cdbfebdd088ebd9a90a394ee296c207ad90 +R c3c34442c09fda273806b002932cda25 U drh -Z 15056c33153d13d2de1255f45fb692fc +Z 4d0ec3d41d82b7fff44b33f0ea314f2b diff --git a/manifest.uuid b/manifest.uuid index 57ede99673..1ee8b65c87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5dd41cdbfebdd088ebd9a90a394ee296c207ad90 \ No newline at end of file +f21d217583c205dc17f98bb4877fd4ed98cefcb1 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b943751253..9af99dd8ad 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5519,9 +5519,15 @@ static void freePage(MemPage *pPage, int *pRC){ } /* -** Free any overflow pages associated with the given Cell. +** Free any overflow pages associated with the given Cell. Write the +** local Cell size (the number of bytes on the original page, omitting +** overflow) into *pnSize. */ -static int clearCell(MemPage *pPage, unsigned char *pCell){ +static int clearCell( + MemPage *pPage, /* The page that contains the Cell */ + unsigned char *pCell, /* First byte of the Cell */ + u16 *pnSize /* Write the size of the Cell here */ +){ BtShared *pBt = pPage->pBt; CellInfo info; Pgno ovflPgno; @@ -5531,6 +5537,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); btreeParseCellPtr(pPage, pCell, &info); + *pnSize = info.nSize; if( info.iOverflow==0 ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } @@ -7166,8 +7173,7 @@ int sqlite3BtreeInsert( if( !pPage->leaf ){ memcpy(newCell, oldCell, 4); } - szOld = cellSizePtr(pPage, oldCell); - rc = clearCell(pPage, oldCell); + rc = clearCell(pPage, oldCell, &szOld); dropCell(pPage, idx, szOld, &rc); if( rc ) goto end_insert; }else if( loc<0 && pPage->nCell>0 ){ @@ -7229,6 +7235,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ unsigned char *pCell; /* Pointer to cell to delete */ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ + u16 szCell; /* Size of the cell being deleted */ assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -7277,8 +7284,8 @@ int sqlite3BtreeDelete(BtCursor *pCur){ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; - rc = clearCell(pPage, pCell); - dropCell(pPage, iCellIdx, cellSizePtr(pPage, pCell), &rc); + rc = clearCell(pPage, pCell, &szCell); + dropCell(pPage, iCellIdx, szCell, &rc); if( rc ) return rc; /* If the cell deleted was not located on a leaf page, then the cursor @@ -7510,6 +7517,7 @@ static int clearDatabasePage( unsigned char *pCell; int i; int hdr; + u16 szCell; assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ @@ -7525,7 +7533,7 @@ static int clearDatabasePage( rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); if( rc ) goto cleardatabasepage_out; } - rc = clearCell(pPage, pCell); + rc = clearCell(pPage, pCell, &szCell); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ From feada2df39af475b18055c117d4f3c75212dbbe5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 13:20:22 +0000 Subject: [PATCH 13/25] Do not allow parameters in a DEFAULT clause of a CREATE TABLE statement. Ticket [78c0c8c3c9f7c1]. FossilOrigin-Name: 1ad2bc1ed4c4ac81ac67a9660761f0eeb47c7fef --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/build.c | 2 +- src/expr.c | 44 +++++++++++++++++++++++++++++++++----------- src/sqliteInt.h | 2 +- test/default.test | 28 ++++++++++++++++++++++++++++ 6 files changed, 73 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 209072a802..01e1d67d9f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\sclearCell()\sroutine\sreturn\sthe\scell\ssize\sto\sthe\scaller,\srather\nthan\shave\sthe\scaller\smake\sa\sseparate\scall\sto\scellSizePtr(). -D 2014-09-24T02:05:41.968 +C Do\snot\sallow\sparameters\sin\sa\sDEFAULT\sclause\sof\sa\sCREATE\sTABLE\sstatement.\nTicket\s[78c0c8c3c9f7c1]. +D 2014-09-24T13:20:22.559 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,13 +175,13 @@ F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c 183d62b37358f95d2ffac796f8491591a3456362 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef -F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 +F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c 4f101c8ddc6d5a22303c88278069f5261562a9a8 +F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c 1629ccdd8ef3f19d7accc9d9287190489469ff81 @@ -232,7 +232,7 @@ F src/shell.c dad23987c34faddb061a339da3e92e05ccc6935e F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 5ecde2191721a94cdce0d5248e26a373e0b17a70 +F src/sqliteInt.h ca777ba045b1849d36a44dee1c71c652ae915093 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -438,7 +438,7 @@ F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test 42973251b9429f2c41b77eb98a7b0b0ba2d3b2c0 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204 F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 -F test/default.test 792c3c70836f1901e2a8cb34fa0880ed71e2c1a9 +F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5dd41cdbfebdd088ebd9a90a394ee296c207ad90 -R c3c34442c09fda273806b002932cda25 +P f21d217583c205dc17f98bb4877fd4ed98cefcb1 +R 0c35c4c692384a20d9dc945800977dcf U drh -Z 4d0ec3d41d82b7fff44b33f0ea314f2b +Z ea463bda0fe2fcb1086a67d4498e377c diff --git a/manifest.uuid b/manifest.uuid index 1ee8b65c87..61739fa20a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f21d217583c205dc17f98bb4877fd4ed98cefcb1 \ No newline at end of file +1ad2bc1ed4c4ac81ac67a9660761f0eeb47c7fef \ No newline at end of file diff --git a/src/build.c b/src/build.c index 0c02a56fe7..777831aab5 100644 --- a/src/build.c +++ b/src/build.c @@ -1236,7 +1236,7 @@ void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ p = pParse->pNewTable; if( p!=0 ){ pCol = &(p->aCol[p->nCol-1]); - if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr) ){ + if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr, db->init.busy) ){ sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", pCol->zName); }else{ diff --git a/src/expr.c b/src/expr.c index c8e8e78268..57e462ea6e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1212,32 +1212,40 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ /* ** These routines are Walker callbacks. Walker.u.pi is a pointer ** to an integer. These routines are checking an expression to see -** if it is a constant. Set *Walker.u.pi to 0 if the expression is +** if it is a constant. Set *Walker.u.i to 0 if the expression is ** not constant. ** ** These callback routines are used to implement the following: ** -** sqlite3ExprIsConstant() -** sqlite3ExprIsConstantNotJoin() -** sqlite3ExprIsConstantOrFunction() +** sqlite3ExprIsConstant() pWalker->u.i==1 +** sqlite3ExprIsConstantNotJoin() pWalker->u.i==2 +** sqlite3ExprIsConstantOrFunction() pWalker->u.i==3 or 4 ** +** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions +** in a CREATE TABLE statement. The Walker.u.i value is 4 when parsing +** an existing schema and 3 when processing a new statement. A bound +** parameter raises an error for new statements, but is silently converted +** to NULL for existing schemas. This allows sqlite_master tables that +** contain a bound parameter because they were generated by older versions +** of SQLite to be parsed by newer versions of SQLite without raising a +** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ - /* If pWalker->u.i is 3 then any term of the expression that comes from + /* If pWalker->u.i is 2 then any term of the expression that comes from ** the ON or USING clauses of a join disqualifies the expression ** from being considered constant. */ - if( pWalker->u.i==3 && ExprHasProperty(pExpr, EP_FromJoin) ){ + if( pWalker->u.i==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ pWalker->u.i = 0; return WRC_Abort; } switch( pExpr->op ){ /* Consider functions to be constant if all their arguments are constant - ** and either pWalker->u.i==2 or the function as the SQLITE_FUNC_CONST + ** and either pWalker->u.i==3 or 4 or the function as the SQLITE_FUNC_CONST ** flag. */ case TK_FUNCTION: - if( pWalker->u.i==2 || ExprHasProperty(pExpr,EP_Constant) ){ + if( pWalker->u.i>=3 || ExprHasProperty(pExpr,EP_Constant) ){ return WRC_Continue; } /* Fall through */ @@ -1251,6 +1259,19 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_AGG_COLUMN ); pWalker->u.i = 0; return WRC_Abort; + case TK_VARIABLE: + if( pWalker->u.i==4 ){ + /* Silently convert bound parameters that appear inside of CREATE + ** statements into a NULL when parsing the CREATE statement text out + ** of the sqlite_master table */ + pExpr->op = TK_NULL; + }else if( pWalker->u.i==3 ){ + /* A bound parameter in a CREATE statement that originates from + ** sqlite3_prepare() causes an error */ + pWalker->u.i = 0; + return WRC_Abort; + } + /* Fall through */ default: testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */ testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */ @@ -1291,7 +1312,7 @@ int sqlite3ExprIsConstant(Expr *p){ ** an ON or USING clause. */ int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 3); + return exprIsConst(p, 2); } /* @@ -1303,8 +1324,9 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){ ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. */ -int sqlite3ExprIsConstantOrFunction(Expr *p){ - return exprIsConst(p, 2); +int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ + assert( isInit==0 || isInit==1 ); + return exprIsConst(p, 3+isInit); } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9a9675b0ab..414f30b66c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3285,7 +3285,7 @@ void sqlite3CloseSavepoints(sqlite3 *); void sqlite3LeaveMutexAndCloseZombie(sqlite3*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); -int sqlite3ExprIsConstantOrFunction(Expr*); +int sqlite3ExprIsConstantOrFunction(Expr*, u8); int sqlite3ExprIsInteger(Expr*, int*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); diff --git a/test/default.test b/test/default.test index d6b6f97d98..406eb53677 100644 --- a/test/default.test +++ b/test/default.test @@ -99,4 +99,32 @@ do_execsql_test default-3.3 { SELECT * FROM t300; } {2147483647 2147483648 9223372036854775807 -2147483647 -2147483648 -9223372036854775808 9.22337203685478e+18 9223372036854775807} +# Do now allow bound parameters in new DEFAULT values. +# Silently convert bound parameters to NULL in DEFAULT causes +# in the sqlite_master table, for backwards compatibility. +# +db close +forcedelete test.db +sqlite3 db test.db +do_execsql_test default-4.0 { + CREATE TABLE t1(a TEXT, b TEXT DEFAULT(99)); + PRAGMA writable_schema=ON; + UPDATE sqlite_master SET sql='CREATE TABLE t1(a TEXT, b TEXT DEFAULT(:xyz))'; +} {} +db close +sqlite3 db test.db +do_execsql_test default-4.1 { + INSERT INTO t1(a) VALUES('xyzzy'); + SELECT a, quote(b) FROM t1; +} {xyzzy NULL} +do_catchsql_test default-4.2 { + CREATE TABLE t2(a TEXT, b TEXT DEFAULT(:xyz)); +} {1 {default value of column [b] is not constant}} +do_catchsql_test default-4.3 { + CREATE TABLE t2(a TEXT, b TEXT DEFAULT(abs(:xyz))); +} {1 {default value of column [b] is not constant}} +do_catchsql_test default-4.4 { + CREATE TABLE t2(a TEXT, b TEXT DEFAULT(98+coalesce(5,:xyz))); +} {1 {default value of column [b] is not constant}} + finish_test From b2325b72df09777cf039ff5036d58047fd0bb2f7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 18:31:07 +0000 Subject: [PATCH 14/25] Small performance and size optimization for btreeUnlockIfUnused(). FossilOrigin-Name: 13c746f85d254475b10c3dd58555acd3bbead0ce --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 01e1d67d9f..c64e305a7f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sparameters\sin\sa\sDEFAULT\sclause\sof\sa\sCREATE\sTABLE\sstatement.\nTicket\s[78c0c8c3c9f7c1]. -D 2014-09-24T13:20:22.559 +C Small\sperformance\sand\ssize\soptimization\sfor\sbtreeUnlockIfUnused(). +D 2014-09-24T18:31:07.339 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 183d62b37358f95d2ffac796f8491591a3456362 +F src/btree.c ee77b1d3a346dd0d581e3d729524243c68ba5b96 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f21d217583c205dc17f98bb4877fd4ed98cefcb1 -R 0c35c4c692384a20d9dc945800977dcf +P 1ad2bc1ed4c4ac81ac67a9660761f0eeb47c7fef +R d6b9d6f16b042c47207345c5727aac20 U drh -Z ea463bda0fe2fcb1086a67d4498e377c +Z 51a889840b720ee888395fd53da3c843 diff --git a/manifest.uuid b/manifest.uuid index 61739fa20a..838ae9d867 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ad2bc1ed4c4ac81ac67a9660761f0eeb47c7fef \ No newline at end of file +13c746f85d254475b10c3dd58555acd3bbead0ce \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9af99dd8ad..d8bf076e85 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2627,11 +2627,11 @@ static void unlockBtreeIfUnused(BtShared *pBt){ assert( sqlite3_mutex_held(pBt->mutex) ); assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ - assert( pBt->pPage1->aData ); + MemPage *pPage1 = pBt->pPage1; + assert( pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); - assert( pBt->pPage1->aData ); - releasePage(pBt->pPage1); pBt->pPage1 = 0; + releasePage(pPage1); } } From 3fbb022b98a05d985571a188003f3dd1fb2b94a5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 19:47:27 +0000 Subject: [PATCH 15/25] Have each open database allocate its pTmpSpace when the first write cursor is opened, rather than on each insert or delete, for a small space savings and performance boost. FossilOrigin-Name: 99323552c001bc9173eb2a44542234c8ef7a9845 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 11 ++++++----- src/vdbe.c | 4 ---- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c64e305a7f..9d16127977 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\sand\ssize\soptimization\sfor\sbtreeUnlockIfUnused(). -D 2014-09-24T18:31:07.339 +C Have\seach\sopen\sdatabase\sallocate\sits\spTmpSpace\swhen\sthe\sfirst\swrite\scursor\nis\sopened,\srather\sthan\son\seach\sinsert\sor\sdelete,\sfor\sa\ssmall\sspace\ssavings\nand\sperformance\sboost. +D 2014-09-24T19:47:27.033 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c ee77b1d3a346dd0d581e3d729524243c68ba5b96 +F src/btree.c 4d5cdfeaea4a00f796c17af246dd5b48cd525d5e F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c de1af1795bebdad20c23e82bafa2f531e9893198 +F src/vdbe.c 23db9b11f0d0a022c42bf71c2b036d32c82a9abd F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1ad2bc1ed4c4ac81ac67a9660761f0eeb47c7fef -R d6b9d6f16b042c47207345c5727aac20 +P 13c746f85d254475b10c3dd58555acd3bbead0ce +R dd216235dc00cda1e50e4687ef88aee3 U drh -Z 51a889840b720ee888395fd53da3c843 +Z 5eb3b3c4f4a76619f8f4a6dc0df5e879 diff --git a/manifest.uuid b/manifest.uuid index 838ae9d867..c31ec3cb1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13c746f85d254475b10c3dd58555acd3bbead0ce \ No newline at end of file +99323552c001bc9173eb2a44542234c8ef7a9845 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index d8bf076e85..37aabbef77 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3672,6 +3672,10 @@ static int btreeCursor( if( NEVER(wrFlag && (pBt->btsFlags & BTS_READ_ONLY)!=0) ){ return SQLITE_READONLY; } + if( wrFlag ){ + allocateTempSpace(pBt); + if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM; + } if( iTable==1 && btreePagecount(pBt)==0 ){ assert( wrFlag==0 ); iTable = 0; @@ -7154,9 +7158,8 @@ int sqlite3BtreeInsert( pCur->pgnoRoot, nKey, nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit ); - allocateTempSpace(pBt); newCell = pBt->pTmpSpace; - if( newCell==0 ) return SQLITE_NOMEM; + assert( newCell!=0 ); rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); if( rc ) goto end_insert; assert( szNew==cellSizePtr(pPage, newCell) ); @@ -7302,10 +7305,8 @@ int sqlite3BtreeDelete(BtCursor *pCur){ pCell = findCell(pLeaf, pLeaf->nCell-1); nCell = cellSizePtr(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); - - allocateTempSpace(pBt); pTmp = pBt->pTmpSpace; - + assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); diff --git a/src/vdbe.c b/src/vdbe.c index 26ca72b9f4..4c2b75f641 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3287,10 +3287,6 @@ case OP_OpenWrite: { assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); sqlite3BtreeCursorHints(pCur->pCursor, (pOp->p5 & OPFLAG_BULKCSR)); - /* Since it performs no memory allocation or IO, the only value that - ** sqlite3BtreeCursor() may return is SQLITE_OK. */ - assert( rc==SQLITE_OK ); - /* Set the VdbeCursor.isTable variable. Previous versions of ** SQLite used to check if the root-page flags were sane at this point ** and report database corruption if they were not, but this check has From 328d913cbdb3aac68c244abbf433ef318f4593a4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 00:56:00 +0000 Subject: [PATCH 16/25] Size reduction and performance improvement in the LIKE and GLOB operators. FossilOrigin-Name: b2c89ef49cd19b8031a8149a2dc47cea07dd04e0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 11 +++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 9d16127977..6a57efd696 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\seach\sopen\sdatabase\sallocate\sits\spTmpSpace\swhen\sthe\sfirst\swrite\scursor\nis\sopened,\srather\sthan\son\seach\sinsert\sor\sdelete,\sfor\sa\ssmall\sspace\ssavings\nand\sperformance\sboost. -D 2014-09-24T19:47:27.033 +C Size\sreduction\sand\sperformance\simprovement\sin\sthe\sLIKE\sand\sGLOB\soperators. +D 2014-09-25T00:56:00.685 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c 1629ccdd8ef3f19d7accc9d9287190489469ff81 +F src/func.c fd49097fdd74eecbc244e5e64fd288a303db20e9 F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 13c746f85d254475b10c3dd58555acd3bbead0ce -R dd216235dc00cda1e50e4687ef88aee3 +P 99323552c001bc9173eb2a44542234c8ef7a9845 +R 81650f65f515728b656a3f04c1aeaf34 U drh -Z 5eb3b3c4f4a76619f8f4a6dc0df5e879 +Z 2d4294ecf245e95fb6c7da3803927f1a diff --git a/manifest.uuid b/manifest.uuid index c31ec3cb1b..e8ae266ea4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -99323552c001bc9173eb2a44542234c8ef7a9845 \ No newline at end of file +b2c89ef49cd19b8031a8149a2dc47cea07dd04e0 \ No newline at end of file diff --git a/src/func.c b/src/func.c index e1961118fd..fc908ded36 100644 --- a/src/func.c +++ b/src/func.c @@ -622,10 +622,9 @@ static int patternCompare( u8 matchAll = pInfo->matchAll; u8 matchSet = pInfo->matchSet; u8 noCase = pInfo->noCase; - int prevEscape = 0; /* True if the previous character was 'escape' */ while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ - if( c==matchAll && !prevEscape ){ + if( c==matchAll ){ while( (c=sqlite3Utf8Read(&zPattern)) == matchAll || c == matchOne ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ @@ -664,7 +663,7 @@ static int patternCompare( if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; - }else if( c==matchOne && !prevEscape ){ + }else if( c==matchOne ){ if( sqlite3Utf8Read(&zString)==0 ){ return 0; } @@ -700,10 +699,11 @@ static int patternCompare( if( c2==0 || (seen ^ invert)==0 ){ return 0; } - }else if( esc==c && !prevEscape ){ - prevEscape = 1; }else{ c2 = sqlite3Utf8Read(&zString); + if( c==esc ){ + c = sqlite3Utf8Read(&zPattern); + } if( noCase ){ GlobUpperToLower(c); GlobUpperToLower(c2); @@ -711,7 +711,6 @@ static int patternCompare( if( c!=c2 ){ return 0; } - prevEscape = 0; } } return *zString==0; From 97348b37c235f53df245e916a5d32652a837168e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 02:44:29 +0000 Subject: [PATCH 17/25] Change that might allow SQLite to build and work using the EBCDIC character set. FossilOrigin-Name: ef30e0352b3d4a29749cd0872c10e45a6649ec52 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/ctime.c | 2 +- src/sqliteInt.h | 2 ++ src/tokenize.c | 1 + 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6a57efd696..de2a2bc27f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Size\sreduction\sand\sperformance\simprovement\sin\sthe\sLIKE\sand\sGLOB\soperators. -D 2014-09-25T00:56:00.685 +C Change\sthat\smight\sallow\sSQLite\sto\sbuild\sand\swork\susing\sthe\sEBCDIC\scharacter\nset. +D 2014-09-25T02:44:29.974 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,7 +178,7 @@ F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 -F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 +F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 @@ -232,7 +232,7 @@ F src/shell.c dad23987c34faddb061a339da3e92e05ccc6935e F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h ca777ba045b1849d36a44dee1c71c652ae915093 +F src/sqliteInt.h 5e09fe04f999223680801ddf8fbae6b60751d613 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -283,7 +283,7 @@ F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff -F src/tokenize.c 3df63041994f55afeb168b463ec836e8f1c50e7c +F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 99323552c001bc9173eb2a44542234c8ef7a9845 -R 81650f65f515728b656a3f04c1aeaf34 +P b2c89ef49cd19b8031a8149a2dc47cea07dd04e0 +R 594413b3f59b58a61e72cebd73f880a5 U drh -Z 2d4294ecf245e95fb6c7da3803927f1a +Z 19c5c0df4d171b8ce37674c647553165 diff --git a/manifest.uuid b/manifest.uuid index e8ae266ea4..7b2fbde36d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2c89ef49cd19b8031a8149a2dc47cea07dd04e0 \ No newline at end of file +ef30e0352b3d4a29749cd0872c10e45a6649ec52 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index 6f7ac8fcba..82a2f35204 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -395,7 +395,7 @@ int sqlite3_compileoption_used(const char *zOptName){ ** linear search is adequate. No need for a binary search. */ for(i=0; inChange */ +#define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ @@ -2987,6 +2988,7 @@ int sqlite3CantopenError(int); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) #endif +int sqlite3IsIdChar(u8); /* ** Internal function prototypes diff --git a/src/tokenize.c b/src/tokenize.c index 8a7894514c..5bb9155460 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -102,6 +102,7 @@ const char sqlite3IsEbcdicIdChar[] = { }; #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) #endif +int sqlite3IsIdChar(u8 c){ return IdChar(c); } /* From 88b3322f272341f4a8145583d6ba55b2d35da211 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 03:51:37 +0000 Subject: [PATCH 18/25] More performance optimization for the LIKE and GLOB operators. FossilOrigin-Name: 5ab1023d6cfe31fa8a194804b8216058977ac973 --- manifest | 12 ++--- manifest.uuid | 2 +- src/func.c | 146 +++++++++++++++++++++++++++----------------------- 3 files changed, 86 insertions(+), 74 deletions(-) diff --git a/manifest b/manifest index de2a2bc27f..37342c612c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthat\smight\sallow\sSQLite\sto\sbuild\sand\swork\susing\sthe\sEBCDIC\scharacter\nset. -D 2014-09-25T02:44:29.974 +C More\sperformance\soptimization\sfor\sthe\sLIKE\sand\sGLOB\soperators. +D 2014-09-25T03:51:37.139 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c fd49097fdd74eecbc244e5e64fd288a303db20e9 +F src/func.c 610b18afde750686785cdad9196b9fb1b03dc9b3 F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b2c89ef49cd19b8031a8149a2dc47cea07dd04e0 -R 594413b3f59b58a61e72cebd73f880a5 +P ef30e0352b3d4a29749cd0872c10e45a6649ec52 +R 8a27395e5bbddced71b289eca8fe4771 U drh -Z 19c5c0df4d171b8ce37674c647553165 +Z c372796d929c11ef7d059e5ca2e0eb4a diff --git a/manifest.uuid b/manifest.uuid index 7b2fbde36d..035ef4eeb2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef30e0352b3d4a29749cd0872c10e45a6649ec52 \ No newline at end of file +5ab1023d6cfe31fa8a194804b8216058977ac973 \ No newline at end of file diff --git a/src/func.c b/src/func.c index fc908ded36..a2a5c1858d 100644 --- a/src/func.c +++ b/src/func.c @@ -567,10 +567,12 @@ struct compareInfo { ** whereas only characters less than 0x80 do in ASCII. */ #if defined(SQLITE_EBCDIC) -# define sqlite3Utf8Read(A) (*((*A)++)) -# define GlobUpperToLower(A) A = sqlite3UpperToLower[A] +# define sqlite3Utf8Read(A) (*((*A)++)) +# define GlobUpperToLower(A) A = sqlite3UpperToLower[A] +# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] #else -# define GlobUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } +# define GlobUpperToLower(A) if( A<=0x7f ){ A = sqlite3UpperToLower[A]; } +# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -618,10 +620,17 @@ static int patternCompare( u32 c, c2; int invert; int seen; - u8 matchOne = pInfo->matchOne; - u8 matchAll = pInfo->matchAll; - u8 matchSet = pInfo->matchSet; - u8 noCase = pInfo->noCase; + u32 matchOne = pInfo->matchOne; + u32 matchAll = pInfo->matchAll; + u32 matchOther; + u8 noCase = pInfo->noCase; + + /* The GLOB operator does not have an ESCAPE clause. And LIKE does not + ** have the matchSet operator. So we either have to look for one or + ** the other, never both. Hence the single variable matchOther is used + ** to store the one we have to look for. + */ + matchOther = esc ? esc : pInfo->matchSet; while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ if( c==matchAll ){ @@ -633,26 +642,26 @@ static int patternCompare( } if( c==0 ){ return 1; - }else if( c==esc ){ - c = sqlite3Utf8Read(&zPattern); - if( c==0 ){ - return 0; + }else if( c==matchOther ){ + if( esc ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return 0; + }else{ + assert( matchOther<0x80 ); /* '[' is a single-byte character */ + while( *zString + && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ + SQLITE_SKIP_UTF8(zString); + } + return *zString!=0; } - }else if( c==matchSet ){ - assert( esc==0 ); /* This is GLOB, not LIKE */ - assert( matchSet<0x80 ); /* '[' is a single-byte character */ - while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ - SQLITE_SKIP_UTF8(zString); - } - return *zString!=0; } while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ - if( noCase ){ + if( noCase && c<0x80 ){ GlobUpperToLower(c2); - GlobUpperToLower(c); + GlobUpperToLowerAscii(c); while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(&zString); - GlobUpperToLower(c2); + do{ c2 = *(zString++); }while( c2>0x7f ); + GlobUpperToLowerAscii(c2); } }else{ while( c2 != 0 && c2 != c ){ @@ -663,55 +672,58 @@ static int patternCompare( if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; - }else if( c==matchOne ){ + } + if( c==matchOne ){ if( sqlite3Utf8Read(&zString)==0 ){ return 0; - } - }else if( c==matchSet ){ - u32 prior_c = 0; - assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ - seen = 0; - invert = 0; - c = sqlite3Utf8Read(&zString); - if( c==0 ) return 0; - c2 = sqlite3Utf8Read(&zPattern); - if( c2=='^' ){ - invert = 1; - c2 = sqlite3Utf8Read(&zPattern); - } - if( c2==']' ){ - if( c==']' ) seen = 1; - c2 = sqlite3Utf8Read(&zPattern); - } - while( c2 && c2!=']' ){ - if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ - c2 = sqlite3Utf8Read(&zPattern); - if( c>=prior_c && c<=c2 ) seen = 1; - prior_c = 0; - }else{ - if( c==c2 ){ - seen = 1; - } - prior_c = c2; - } - c2 = sqlite3Utf8Read(&zPattern); - } - if( c2==0 || (seen ^ invert)==0 ){ - return 0; - } - }else{ - c2 = sqlite3Utf8Read(&zString); - if( c==esc ){ - c = sqlite3Utf8Read(&zPattern); - } - if( noCase ){ - GlobUpperToLower(c); - GlobUpperToLower(c2); - } - if( c!=c2 ){ - return 0; + }else{ + continue; } } + if( c==matchOther ){ + if( esc ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return 0; + }else{ + u32 prior_c = 0; + seen = 0; + invert = 0; + c = sqlite3Utf8Read(&zString); + if( c==0 ) return 0; + c2 = sqlite3Utf8Read(&zPattern); + if( c2=='^' ){ + invert = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ + c2 = sqlite3Utf8Read(&zPattern); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==0 || (seen ^ invert)==0 ){ + return 0; + } + continue; + } + } + c2 = sqlite3Utf8Read(&zString); + if( c==c2 ) continue; + if( !noCase ) return 0; + GlobUpperToLower(c); + GlobUpperToLower(c2); + if( c!=c2 ) return 0; } return *zString==0; } From 9fdfdc893bb14459ed6cfc142ca7208ecdb3abc8 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 11:08:57 +0000 Subject: [PATCH 19/25] Still more performance enhancements to the LIKE and GLOB operators. FossilOrigin-Name: 6c8924cacc2b875270770fed2cc3b1884f57a655 --- manifest | 12 +++---- manifest.uuid | 2 +- src/func.c | 98 ++++++++++++++++++++++++++++++--------------------- 3 files changed, 65 insertions(+), 47 deletions(-) diff --git a/manifest b/manifest index 37342c612c..fb9c0ed010 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\sperformance\soptimization\sfor\sthe\sLIKE\sand\sGLOB\soperators. -D 2014-09-25T03:51:37.139 +C Still\smore\sperformance\senhancements\sto\sthe\sLIKE\sand\sGLOB\soperators. +D 2014-09-25T11:08:57.081 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c 610b18afde750686785cdad9196b9fb1b03dc9b3 +F src/func.c 727a324e87a3392a47e44568b901d2fb96ba0ed4 F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef30e0352b3d4a29749cd0872c10e45a6649ec52 -R 8a27395e5bbddced71b289eca8fe4771 +P 5ab1023d6cfe31fa8a194804b8216058977ac973 +R 3c45ffccf6b8b9761c0cc8bd190b6e11 U drh -Z c372796d929c11ef7d059e5ca2e0eb4a +Z 88a762d5e9ae73a0ed674e2385a2b544 diff --git a/manifest.uuid b/manifest.uuid index 035ef4eeb2..9939e8087e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ab1023d6cfe31fa8a194804b8216058977ac973 \ No newline at end of file +6c8924cacc2b875270770fed2cc3b1884f57a655 \ No newline at end of file diff --git a/src/func.c b/src/func.c index a2a5c1858d..5b7056b401 100644 --- a/src/func.c +++ b/src/func.c @@ -585,7 +585,7 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; /* ** Compare two UTF-8 strings for equality where the first string can -** potentially be a "glob" expression. Return true (1) if they +** potentially be a "glob" or "like" expression. Return true (1) if they ** are the same and false (0) if they are different. ** ** Globbing rules: @@ -605,11 +605,18 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; ** "[a-z]" matches any single lower-case letter. To match a '-', make ** it the last character in the list. ** +** Like matching rules: +** +** '%' Matches any sequence of zero or more characters +** +*** '_' Matches any one character +** +** Ec Where E is the "esc" character and c is any other +** character, including '%', '_', and esc, match exactly c. +** +** The comments through this routine usually assume glob matching. +** ** This routine is usually quick, but can be N**2 in the worst case. -** -** Hints: to match '*' or '?', put them in "[]". Like this: -** -** abc[*]xyz Matches "abc*xyz" only */ static int patternCompare( const u8 *zPattern, /* The glob pattern */ @@ -617,13 +624,12 @@ static int patternCompare( const struct compareInfo *pInfo, /* Information about how to do the compare */ u32 esc /* The escape character */ ){ - u32 c, c2; - int invert; - int seen; - u32 matchOne = pInfo->matchOne; - u32 matchAll = pInfo->matchAll; - u32 matchOther; - u8 noCase = pInfo->noCase; + u32 c, c2; /* Next pattern and input string chars */ + u32 matchOne = pInfo->matchOne; /* "?" or "_" */ + u32 matchAll = pInfo->matchAll; /* "*" or "%" */ + u32 matchOther; /* "[" or the escape character */ + u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ + const u8 *zEscaped = 0; /* One past the last escaped input char */ /* The GLOB operator does not have an ESCAPE clause. And LIKE does not ** have the matchSet operator. So we either have to look for one or @@ -633,7 +639,10 @@ static int patternCompare( matchOther = esc ? esc : pInfo->matchSet; while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ - if( c==matchAll ){ + if( c==matchAll ){ /* Match "*" */ + /* Skip over multiple "*" characters in the pattern. If there + ** are also "?" characters, skip those as well, but consume a + ** single character of the input string for each "?" skipped */ while( (c=sqlite3Utf8Read(&zPattern)) == matchAll || c == matchOne ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ @@ -641,12 +650,14 @@ static int patternCompare( } } if( c==0 ){ - return 1; + return 1; /* "*" at the end of the pattern matches */ }else if( c==matchOther ){ if( esc ){ c = sqlite3Utf8Read(&zPattern); if( c==0 ) return 0; }else{ + /* "[...]" immediately follows the "*". We have to do a slow + ** recursive search in this case, but it is an unusual case. */ assert( matchOther<0x80 ); /* '[' is a single-byte character */ while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ @@ -655,39 +666,45 @@ static int patternCompare( return *zString!=0; } } - while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ - if( noCase && c<0x80 ){ - GlobUpperToLower(c2); - GlobUpperToLowerAscii(c); - while( c2 != 0 && c2 != c ){ - do{ c2 = *(zString++); }while( c2>0x7f ); - GlobUpperToLowerAscii(c2); - } + + /* At this point variable c contains the first character of the + ** pattern string past the "*". Search in the input string for the + ** first matching character and recursively contine the match from + ** that point. + ** + ** For a case-insensitive search, set variable cx to be the same as + ** c but in the other case and search the input string for either + ** c or cx. + */ + if( c<=0x80 ){ + u32 cx; + if( noCase ){ + cx = sqlite3Toupper(c); + c = sqlite3Tolower(c); }else{ - while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(&zString); - } + cx = c; + } + while( (c2 = *(zString++))!=0 ){ + if( c2!=c && c2!=cx ) continue; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; + } + }else{ + while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ + if( c2!=c ) continue; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } - if( c2==0 ) return 0; - if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; } - if( c==matchOne ){ - if( sqlite3Utf8Read(&zString)==0 ){ - return 0; - }else{ - continue; - } - } if( c==matchOther ){ if( esc ){ c = sqlite3Utf8Read(&zPattern); if( c==0 ) return 0; + zEscaped = zPattern; }else{ u32 prior_c = 0; - seen = 0; - invert = 0; + int seen = 0; + int invert = 0; c = sqlite3Utf8Read(&zString); if( c==0 ) return 0; c2 = sqlite3Utf8Read(&zPattern); @@ -720,10 +737,11 @@ static int patternCompare( } c2 = sqlite3Utf8Read(&zString); if( c==c2 ) continue; - if( !noCase ) return 0; - GlobUpperToLower(c); - GlobUpperToLower(c2); - if( c!=c2 ) return 0; + if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){ + continue; + } + if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; + return 0; } return *zString==0; } From 2c4dc635a198787e14f2548ddd04d9fade2ab278 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 12:31:28 +0000 Subject: [PATCH 20/25] Simplification to the random rowid picking logic that begins running when the maximum possible rowid has already been used. FossilOrigin-Name: 1330c72e172324c68ab49e5bb2ceba985935ae01 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 22 ++++++---------------- test/rowid.test | 4 ++-- 4 files changed, 16 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index fb9c0ed010..658142eff5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Still\smore\sperformance\senhancements\sto\sthe\sLIKE\sand\sGLOB\soperators. -D 2014-09-25T11:08:57.081 +C Simplification\sto\sthe\srandom\srowid\spicking\slogic\sthat\sbegins\srunning\swhen\nthe\smaximum\spossible\srowid\shas\salready\sbeen\sused. +D 2014-09-25T12:31:28.476 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 23db9b11f0d0a022c42bf71c2b036d32c82a9abd +F src/vdbe.c 9fe630d05840aa151a5ba9039901478d9524120b F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -784,7 +784,7 @@ F test/releasetest.tcl a0df0dfc5e3ee83ade87b9cc96db41b52d590b9e F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 -F test/rowid.test b78b30afb9537a73788ca1233a23a32190a3bb1f +F test/rowid.test 9ffee168c4be901820bf5cf5fcbb2105117d0d45 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 F test/savepoint.test 51d3900dc071a7c2ad4248578a5925631b476313 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5ab1023d6cfe31fa8a194804b8216058977ac973 -R 3c45ffccf6b8b9761c0cc8bd190b6e11 +P 6c8924cacc2b875270770fed2cc3b1884f57a655 +R c9d002c28940b83ace850b74161b9f39 U drh -Z 88a762d5e9ae73a0ed674e2385a2b544 +Z a6a7b4b719a1f0df211ae0686c0976a9 diff --git a/manifest.uuid b/manifest.uuid index 9939e8087e..5e3b55899b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c8924cacc2b875270770fed2cc3b1884f57a655 \ No newline at end of file +1330c72e172324c68ab49e5bb2ceba985935ae01 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 4c2b75f641..27d7e74901 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4020,25 +4020,15 @@ case OP_NewRowid: { /* out2-prerelease */ ** it finds one that is not previously used. */ assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is ** an AUTOINCREMENT table. */ - /* on the first attempt, simply do one more than previous */ - v = lastRowid; - v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ - v++; /* ensure non-zero */ cnt = 0; - while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, + do{ + sqlite3_randomness(sizeof(v), &v); + v &= (MAX_ROWID>>1); + v++; + }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, 0, &res))==SQLITE_OK) && (res==0) - && (++cnt<100)){ - /* collision - try another random rowid */ - sqlite3_randomness(sizeof(v), &v); - if( cnt<5 ){ - /* try "small" random rowids for the initial attempts */ - v &= 0xffffff; - }else{ - v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ - } - v++; /* ensure non-zero */ - } + && (++cnt<100)); if( rc==SQLITE_OK && res==0 ){ rc = SQLITE_FULL; /* IMP: R-38219-53002 */ goto abort_due_to_error; diff --git a/test/rowid.test b/test/rowid.test index 6d068d79bb..d232139ea0 100644 --- a/test/rowid.test +++ b/test/rowid.test @@ -679,9 +679,9 @@ do_test rowid-12.2 { save_prng_state execsql { INSERT INTO t7 VALUES(NULL,'b'); - SELECT x, y FROM t7; + SELECT x, y FROM t7 ORDER BY x; } -} {1 b 9223372036854775807 a} +} {/\d+ b 9223372036854775807 a/} execsql {INSERT INTO t7 VALUES(2,'y');} for {set i 1} {$i<100} {incr i} { do_test rowid-12.3.$i { From a15cc47f81fc9fe637166e8ea475e8d81557e0c7 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 13:17:30 +0000 Subject: [PATCH 21/25] Simplifications to the SQL function and aggregate calling procedures. FossilOrigin-Name: 3467049a1705b49905ea88a5c6becb6fe318f2fa --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/func.c | 5 ++++- src/vdbe.c | 18 ++---------------- src/vdbeInt.h | 1 - 5 files changed, 15 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 658142eff5..0b87bd95d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sto\sthe\srandom\srowid\spicking\slogic\sthat\sbegins\srunning\swhen\nthe\smaximum\spossible\srowid\shas\salready\sbeen\sused. -D 2014-09-25T12:31:28.476 +C Simplifications\sto\sthe\sSQL\sfunction\sand\saggregate\scalling\sprocedures. +D 2014-09-25T13:17:30.283 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c 727a324e87a3392a47e44568b901d2fb96ba0ed4 +F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -289,9 +289,9 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 9fe630d05840aa151a5ba9039901478d9524120b +F src/vdbe.c 73eace757ead9fee63576e8c9f5337edb4c8c69d F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 +F src/vdbeInt.h bb7f7ecfdead1a2ae0251b59f86f5724838d975c F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d F src/vdbeaux.c a05adc3c96abdaf3db14768ddd63132fc9678060 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6c8924cacc2b875270770fed2cc3b1884f57a655 -R c9d002c28940b83ace850b74161b9f39 +P 1330c72e172324c68ab49e5bb2ceba985935ae01 +R db6a54c99d97f1320af4a50fd756c282 U drh -Z a6a7b4b719a1f0df211ae0686c0976a9 +Z 1da03df9544819e54ad17b4099a36954 diff --git a/manifest.uuid b/manifest.uuid index 5e3b55899b..d95b1f3d63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1330c72e172324c68ab49e5bb2ceba985935ae01 \ No newline at end of file +3467049a1705b49905ea88a5c6becb6fe318f2fa \ No newline at end of file diff --git a/src/func.c b/src/func.c index 5b7056b401..cf556e2439 100644 --- a/src/func.c +++ b/src/func.c @@ -22,7 +22,10 @@ ** Return the collating function associated with a function. */ static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ - return context->pColl; + VdbeOp *pOp = &context->pVdbe->aOp[context->iOp-1]; + assert( pOp->opcode==OP_CollSeq ); + assert( pOp->p4type==P4_COLLSEQ ); + return pOp->p4.pColl; } /* diff --git a/src/vdbe.c b/src/vdbe.c index 27d7e74901..8ae56416f2 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1557,17 +1557,8 @@ case OP_Function: { ctx.iOp = pc; ctx.pVdbe = p; MemSetTypeFlag(ctx.pOut, MEM_Null); - ctx.fErrorOrAux = 0; - if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>aOp ); - assert( pOp[-1].p4type==P4_COLLSEQ ); - assert( pOp[-1].opcode==OP_CollSeq ); - ctx.pColl = pOp[-1].p4.pColl; - } - db->lastRowid = lastRowid; (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; /* If the function returned an error, throw an exception */ if( ctx.fErrorOrAux ){ @@ -5624,14 +5615,9 @@ case OP_AggStep: { sqlite3VdbeMemInit(&t, db, MEM_Null); ctx.pOut = &t; ctx.isError = 0; - ctx.pColl = 0; + ctx.pVdbe = p; + ctx.iOp = pc; ctx.skipFlag = 0; - if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>p->aOp ); - assert( pOp[-1].p4type==P4_COLLSEQ ); - assert( pOp[-1].opcode==OP_CollSeq ); - ctx.pColl = pOp[-1].p4.pColl; - } (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */ if( ctx.isError ){ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&t)); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 56b5db7d1a..f54c9c6d3b 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -272,7 +272,6 @@ struct sqlite3_context { Mem *pOut; /* The return value is stored here */ FuncDef *pFunc; /* Pointer to function information */ Mem *pMem; /* Memory cell used to store aggregate context */ - CollSeq *pColl; /* Collating sequence */ Vdbe *pVdbe; /* The VM that owns this context */ int iOp; /* Instruction number of OP_Function */ int isError; /* Error code returned by the function. */ From d86334619497f717dd81d70f04999a466f967646 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 17:42:41 +0000 Subject: [PATCH 22/25] Minor code reformatting and comment change, to improve clarity. No logic changes. FossilOrigin-Name: baeb72a356d73e6f624edacd2986ab766105e177 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0b87bd95d0..513f9c3310 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplifications\sto\sthe\sSQL\sfunction\sand\saggregate\scalling\sprocedures. -D 2014-09-25T13:17:30.283 +C Minor\scode\sreformatting\sand\scomment\schange,\sto\simprove\sclarity.\nNo\slogic\schanges. +D 2014-09-25T17:42:41.230 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 73eace757ead9fee63576e8c9f5337edb4c8c69d +F src/vdbe.c e4df7191abdae132b89c57086d0dd1a0d6cb400b F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h bb7f7ecfdead1a2ae0251b59f86f5724838d975c F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1330c72e172324c68ab49e5bb2ceba985935ae01 -R db6a54c99d97f1320af4a50fd756c282 +P 3467049a1705b49905ea88a5c6becb6fe318f2fa +R ed3d6d35b7bb71031f12c63341a22b21 U drh -Z 1da03df9544819e54ad17b4099a36954 +Z c1b4b9aea98707eadcd6494d5019f9dc diff --git a/manifest.uuid b/manifest.uuid index d95b1f3d63..a290ea0464 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3467049a1705b49905ea88a5c6becb6fe318f2fa \ No newline at end of file +baeb72a356d73e6f624edacd2986ab766105e177 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 8ae56416f2..23d18b5031 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4014,8 +4014,7 @@ case OP_NewRowid: { /* out2-prerelease */ cnt = 0; do{ sqlite3_randomness(sizeof(v), &v); - v &= (MAX_ROWID>>1); - v++; + v &= (MAX_ROWID>>1); v++; /* Ensure that v is greater than zero */ }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, 0, &res))==SQLITE_OK) && (res==0) From 3b130beb159f82cc1693fab0a1e924f5b7d3a586 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Sep 2014 01:10:02 +0000 Subject: [PATCH 23/25] If an SQL function makes a recursive call to do an INSERT into the same database, make sure that the last_insert_rowid() for that INSERT is recorded. FossilOrigin-Name: e93aecc090c2a1d3c231bb2bde044886eff0bdf7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 1 + test/rowid.test | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 513f9c3310..cb7c310836 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scode\sreformatting\sand\scomment\schange,\sto\simprove\sclarity.\nNo\slogic\schanges. -D 2014-09-25T17:42:41.230 +C If\san\sSQL\sfunction\smakes\sa\srecursive\scall\sto\sdo\san\sINSERT\sinto\sthe\ssame\ndatabase,\smake\ssure\sthat\sthe\slast_insert_rowid()\sfor\sthat\sINSERT\sis\srecorded. +D 2014-09-26T01:10:02.814 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c e4df7191abdae132b89c57086d0dd1a0d6cb400b +F src/vdbe.c 91b7e12bca7b6056574ce28935e3e3f4769ce3c4 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h bb7f7ecfdead1a2ae0251b59f86f5724838d975c F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -784,7 +784,7 @@ F test/releasetest.tcl a0df0dfc5e3ee83ade87b9cc96db41b52d590b9e F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 -F test/rowid.test 9ffee168c4be901820bf5cf5fcbb2105117d0d45 +F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 F test/savepoint.test 51d3900dc071a7c2ad4248578a5925631b476313 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3467049a1705b49905ea88a5c6becb6fe318f2fa -R ed3d6d35b7bb71031f12c63341a22b21 +P baeb72a356d73e6f624edacd2986ab766105e177 +R d5c71b678da0df654e28e5676923f229 U drh -Z c1b4b9aea98707eadcd6494d5019f9dc +Z 33ff9c5d364d8f96a21ad755540365a5 diff --git a/manifest.uuid b/manifest.uuid index a290ea0464..a544dd408b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -baeb72a356d73e6f624edacd2986ab766105e177 \ No newline at end of file +e93aecc090c2a1d3c231bb2bde044886eff0bdf7 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 23d18b5031..1a1e441cd4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1559,6 +1559,7 @@ case OP_Function: { MemSetTypeFlag(ctx.pOut, MEM_Null); ctx.fErrorOrAux = 0; (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ + lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */ /* If the function returned an error, throw an exception */ if( ctx.fErrorOrAux ){ diff --git a/test/rowid.test b/test/rowid.test index d232139ea0..b00b5287fd 100644 --- a/test/rowid.test +++ b/test/rowid.test @@ -701,5 +701,19 @@ do_test rowid-12.4 { } } {1 {database or disk is full}} +# INSERTs that happen inside of nested function calls are recorded +# by last_insert_rowid. +# +proc rowid_addrow_func {n} { + db eval {INSERT INTO t13(rowid,x) VALUES($n,$n*$n)} + return [db last_insert_rowid] +} +db function addrow rowid_addrow_func +do_execsql_test rowid-13.1 { + CREATE TABLE t13(x); + INSERT INTO t13(rowid,x) VALUES(1234,5); + SELECT rowid, x, addrow(rowid+1000), '|' FROM t13 LIMIT 3; + SELECT last_insert_rowid(); +} {1234 5 2234 | 2234 4990756 3234 | 3234 10458756 4234 | 4234} finish_test From 867db8315930541fa5f33da083066d56a6eb06b2 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Sep 2014 02:41:05 +0000 Subject: [PATCH 24/25] Fix the "PRAGMA integrity_check" command so that it avoids formatting error message context messages until it actually needs to generate an error message. This avoids much formatting, and hence greatly improves the performance of "PRAGMA integrity_check" in the common case when there are no errors. It also makes the code a little smaller. FossilOrigin-Name: 83913515830aa850f9e38406f9422d7e88dcab66 --- manifest | 14 +++--- manifest.uuid | 2 +- src/btree.c | 129 ++++++++++++++++++++++++++++--------------------- src/btreeInt.h | 2 + 4 files changed, 84 insertions(+), 63 deletions(-) diff --git a/manifest b/manifest index cb7c310836..131d77b3a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\san\sSQL\sfunction\smakes\sa\srecursive\scall\sto\sdo\san\sINSERT\sinto\sthe\ssame\ndatabase,\smake\ssure\sthat\sthe\slast_insert_rowid()\sfor\sthat\sINSERT\sis\srecorded. -D 2014-09-26T01:10:02.814 +C Fix\sthe\s"PRAGMA\sintegrity_check"\scommand\sso\sthat\sit\savoids\sformatting\serror\nmessage\scontext\smessages\suntil\sit\sactually\sneeds\sto\sgenerate\san\serror\smessage.\nThis\savoids\smuch\sformatting,\sand\shence\sgreatly\simproves\sthe\sperformance\sof\n"PRAGMA\sintegrity_check"\sin\sthe\scommon\scase\swhen\sthere\sare\sno\serrors.\s\sIt\salso\nmakes\sthe\scode\sa\slittle\ssmaller. +D 2014-09-26T02:41:05.726 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,9 +172,9 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 4d5cdfeaea4a00f796c17af246dd5b48cd525d5e +F src/btree.c 59f03e421dad3cb6e27cc7d2393d3a7459be4b5e F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 -F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef +F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P baeb72a356d73e6f624edacd2986ab766105e177 -R d5c71b678da0df654e28e5676923f229 +P e93aecc090c2a1d3c231bb2bde044886eff0bdf7 +R 32be19747fff5e8f2465eed0f224b45d U drh -Z 33ff9c5d364d8f96a21ad755540365a5 +Z f56b9000203c19d0f3a8172e8374b279 diff --git a/manifest.uuid b/manifest.uuid index a544dd408b..b182c8c316 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e93aecc090c2a1d3c231bb2bde044886eff0bdf7 \ No newline at end of file +83913515830aa850f9e38406f9422d7e88dcab66 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 37aabbef77..135b40139f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7880,11 +7880,11 @@ Pager *sqlite3BtreePager(Btree *p){ */ static void checkAppendMsg( IntegrityCk *pCheck, - char *zMsg1, const char *zFormat, ... ){ va_list ap; + char zBuf[200]; if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; @@ -7892,8 +7892,9 @@ static void checkAppendMsg( if( pCheck->errMsg.nChar ){ sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); } - if( zMsg1 ){ - sqlite3StrAccumAppendAll(&pCheck->errMsg, zMsg1); + if( pCheck->zPfx ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, pCheck->zPfx, pCheck->v1, pCheck->v2); + sqlite3StrAccumAppendAll(&pCheck->errMsg, zBuf); } sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); va_end(ap); @@ -7931,14 +7932,14 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** ** Also check that the page number is in bounds. */ -static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){ +static int checkRef(IntegrityCk *pCheck, Pgno iPage){ if( iPage==0 ) return 1; if( iPage>pCheck->nPage ){ - checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage); + checkAppendMsg(pCheck, "invalid page number %d", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ - checkAppendMsg(pCheck, zContext, "2nd reference to page %d", iPage); + checkAppendMsg(pCheck, "2nd reference to page %d", iPage); return 1; } setPageReferenced(pCheck, iPage); @@ -7955,8 +7956,7 @@ static void checkPtrmap( IntegrityCk *pCheck, /* Integrity check context */ Pgno iChild, /* Child page number */ u8 eType, /* Expected pointer map type */ - Pgno iParent, /* Expected pointer map parent page number */ - char *zContext /* Context description (used for error msg) */ + Pgno iParent /* Expected pointer map parent page number */ ){ int rc; u8 ePtrmapType; @@ -7965,12 +7965,12 @@ static void checkPtrmap( rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->mallocFailed = 1; - checkAppendMsg(pCheck, zContext, "Failed to read ptrmap key=%d", iChild); + checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); return; } if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", iChild, eType, iParent, ePtrmapType, iPtrmapParent); } @@ -7985,8 +7985,7 @@ static void checkList( IntegrityCk *pCheck, /* Integrity checking context */ int isFreeList, /* True for a freelist. False for overflow page list */ int iPage, /* Page number for first page in the list */ - int N, /* Expected number of pages in the list */ - char *zContext /* Context for error messages */ + int N /* Expected number of pages in the list */ ){ int i; int expected = N; @@ -7995,14 +7994,14 @@ static void checkList( DbPage *pOvflPage; unsigned char *pOvflData; if( iPage<1 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "%d of %d pages missing from overflow list starting at %d", N+1, expected, iFirst); break; } - if( checkRef(pCheck, iPage, zContext) ) break; + if( checkRef(pCheck, iPage) ) break; if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage) ){ - checkAppendMsg(pCheck, zContext, "failed to get page %d", iPage); + checkAppendMsg(pCheck, "failed to get page %d", iPage); break; } pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); @@ -8010,11 +8009,11 @@ static void checkList( int n = get4byte(&pOvflData[4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ - checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); + checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0); } #endif if( n>(int)pCheck->pBt->usableSize/4-2 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "freelist leaf count too big on page %d", iPage); N--; }else{ @@ -8022,10 +8021,10 @@ static void checkList( Pgno iFreePage = get4byte(&pOvflData[8+i*4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ - checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0, zContext); + checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0); } #endif - checkRef(pCheck, iFreePage, zContext); + checkRef(pCheck, iFreePage); } N -= n; } @@ -8038,7 +8037,7 @@ static void checkList( */ if( pCheck->pBt->autoVacuum && N>0 ){ i = get4byte(pOvflData); - checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage, zContext); + checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage); } } #endif @@ -8070,7 +8069,6 @@ static void checkList( static int checkTreePage( IntegrityCk *pCheck, /* Context for the sanity check */ int iPage, /* Page number of the page to check */ - char *zParentContext, /* Parent context */ i64 *pnParentMinKey, i64 *pnParentMaxKey ){ @@ -8081,23 +8079,26 @@ static int checkTreePage( u8 *data; BtShared *pBt; int usableSize; - char zContext[100]; char *hit = 0; i64 nMinKey = 0; i64 nMaxKey = 0; - - sqlite3_snprintf(sizeof(zContext), zContext, "Page %d: ", iPage); + const char *saved_zPfx = pCheck->zPfx; + int saved_v1 = pCheck->v1; + int saved_v2 = pCheck->v2; /* Check that the page exists */ pBt = pCheck->pBt; usableSize = pBt->usableSize; if( iPage==0 ) return 0; - if( checkRef(pCheck, iPage, zParentContext) ) return 0; + if( checkRef(pCheck, iPage) ) return 0; + pCheck->zPfx = "Page %d: "; + pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); - return 0; + depth = -1; + goto end_of_check; } /* Clear MemPage.isInit to make sure the corruption detection code in @@ -8105,10 +8106,11 @@ static int checkTreePage( pPage->isInit = 0; if( (rc = btreeInitPage(pPage))!=0 ){ assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "btreeInitPage() returns error code %d", rc); releasePage(pPage); - return 0; + depth = -1; + goto end_of_check; } /* Check out all the cells. @@ -8121,8 +8123,9 @@ static int checkTreePage( /* Check payload overflow pages */ - sqlite3_snprintf(sizeof(zContext), zContext, - "On tree page %d cell %d: ", iPage, i); + pCheck->zPfx = "On tree page %d cell %d: "; + pCheck->v1 = iPage; + pCheck->v2 = i; pCell = findCell(pPage,i); btreeParseCellPtr(pPage, pCell, &info); sz = info.nPayload; @@ -8132,7 +8135,7 @@ static int checkTreePage( if( i==0 ){ nMinKey = nMaxKey = info.nKey; }else if( info.nKey <= nMaxKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); } nMaxKey = info.nKey; @@ -8144,10 +8147,10 @@ static int checkTreePage( Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage, zContext); + checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage); } #endif - checkList(pCheck, 0, pgnoOvfl, nPage, zContext); + checkList(pCheck, 0, pgnoOvfl, nPage); } /* Check sanity of left child page. @@ -8156,12 +8159,12 @@ static int checkTreePage( pgno = get4byte(pCell); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif - d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0?NULL:&nMaxKey); + d2 = checkTreePage(pCheck, pgno, &nMinKey, i==0?NULL:&nMaxKey); if( i>0 && d2!=depth ){ - checkAppendMsg(pCheck, zContext, "Child page depth differs"); + checkAppendMsg(pCheck, "Child page depth differs"); } depth = d2; } @@ -8169,37 +8172,39 @@ static int checkTreePage( if( !pPage->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - sqlite3_snprintf(sizeof(zContext), zContext, - "On page %d at right child: ", iPage); + pCheck->zPfx = "On page %d at right child: "; + pCheck->v1 = iPage; #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif - checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell?NULL:&nMaxKey); + checkTreePage(pCheck, pgno, NULL, !pPage->nCell?NULL:&nMaxKey); } /* For intKey leaf pages, check that the min/max keys are in order ** with any left/parent/right pages. */ + pCheck->zPfx = "Page %d: "; + pCheck->v1 = iPage; if( pPage->leaf && pPage->intKey ){ /* if we are a left child page */ if( pnParentMinKey ){ /* if we are the left most child page */ if( !pnParentMaxKey ){ if( nMaxKey > *pnParentMinKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (max larger than parent min of %lld)", nMaxKey, *pnParentMinKey); } }else{ if( nMinKey <= *pnParentMinKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (min less than parent min of %lld)", nMinKey, *pnParentMinKey); } if( nMaxKey > *pnParentMaxKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (max larger than parent max of %lld)", nMaxKey, *pnParentMaxKey); } @@ -8208,7 +8213,7 @@ static int checkTreePage( /* else if we're a right child page */ } else if( pnParentMaxKey ){ if( nMinKey <= *pnParentMaxKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (min less than parent max of %lld)", nMinKey, *pnParentMaxKey); } @@ -8220,6 +8225,7 @@ static int checkTreePage( data = pPage->aData; hdr = pPage->hdrOffset; hit = sqlite3PageMalloc( pBt->pageSize ); + pCheck->zPfx = 0; if( hit==0 ){ pCheck->mallocFailed = 1; }else{ @@ -8237,7 +8243,8 @@ static int checkTreePage( size = cellSizePtr(pPage, &data[pc]); } if( (int)(pc+size-1)>=usableSize ){ - checkAppendMsg(pCheck, 0, + pCheck->zPfx = 0; + checkAppendMsg(pCheck, "Corruption detected in cell %d on page %d",i,iPage); }else{ for(j=pc+size-1; j>=pc; j--) hit[j]++; @@ -8259,19 +8266,24 @@ static int checkTreePage( if( hit[i]==0 ){ cnt++; }else if( hit[i]>1 ){ - checkAppendMsg(pCheck, 0, + checkAppendMsg(pCheck, "Multiple uses for byte %d of page %d", i, iPage); break; } } if( cnt!=data[hdr+7] ){ - checkAppendMsg(pCheck, 0, + checkAppendMsg(pCheck, "Fragmentation of %d bytes reported as %d on page %d", cnt, data[hdr+7], iPage); } } sqlite3PageFree(hit); releasePage(pPage); + +end_of_check: + pCheck->zPfx = saved_zPfx; + pCheck->v1 = saved_v1; + pCheck->v2 = saved_v2; return depth+1; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -8312,6 +8324,9 @@ char *sqlite3BtreeIntegrityCheck( sCheck.mxErr = mxErr; sCheck.nErr = 0; sCheck.mallocFailed = 0; + sCheck.zPfx = 0; + sCheck.v1 = 0; + sCheck.v2 = 0; *pnErr = 0; if( sCheck.nPage==0 ){ sqlite3BtreeLeave(p); @@ -8331,8 +8346,10 @@ char *sqlite3BtreeIntegrityCheck( /* Check the integrity of the freelist */ + sCheck.zPfx = "Main freelist: "; checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), - get4byte(&pBt->pPage1->aData[36]), "Main freelist: "); + get4byte(&pBt->pPage1->aData[36])); + sCheck.zPfx = 0; /* Check all the tables. */ @@ -8340,10 +8357,12 @@ char *sqlite3BtreeIntegrityCheck( if( aRoot[i]==0 ) continue; #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum && aRoot[i]>1 ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); } #endif - checkTreePage(&sCheck, aRoot[i], "List of tree roots: ", NULL, NULL); + sCheck.zPfx = "List of tree roots: "; + checkTreePage(&sCheck, aRoot[i], NULL, NULL); + sCheck.zPfx = 0; } /* Make sure every page in the file is referenced @@ -8351,7 +8370,7 @@ char *sqlite3BtreeIntegrityCheck( for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ - checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %d is never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain @@ -8359,11 +8378,11 @@ char *sqlite3BtreeIntegrityCheck( */ if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %d is never used", i); } if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i); + checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); } #endif } @@ -8373,7 +8392,7 @@ char *sqlite3BtreeIntegrityCheck( ** of the integrity check. */ if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ - checkAppendMsg(&sCheck, 0, + checkAppendMsg(&sCheck, "Outstanding page count goes from %d to %d during this analysis", nRef, sqlite3PagerRefcount(pBt->pPager) ); diff --git a/src/btreeInt.h b/src/btreeInt.h index df9684e8e9..9f648fceb0 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -657,6 +657,8 @@ struct IntegrityCk { int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ int mallocFailed; /* A memory allocation error has occurred */ + const char *zPfx; /* Error message prefix */ + int v1, v2; /* Values for up to two %d fields in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ }; From 843e4cdea9418748de18aa76879d69de0e861027 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 26 Sep 2014 18:30:11 +0000 Subject: [PATCH 25/25] Add an assert() to verify the last-row-id for the database just prior to calling a SQL function. FossilOrigin-Name: d026f0c944ce812732d3595eaa3c5d432a86c7dd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 131d77b3a9..63e97cbf9f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s"PRAGMA\sintegrity_check"\scommand\sso\sthat\sit\savoids\sformatting\serror\nmessage\scontext\smessages\suntil\sit\sactually\sneeds\sto\sgenerate\san\serror\smessage.\nThis\savoids\smuch\sformatting,\sand\shence\sgreatly\simproves\sthe\sperformance\sof\n"PRAGMA\sintegrity_check"\sin\sthe\scommon\scase\swhen\sthere\sare\sno\serrors.\s\sIt\salso\nmakes\sthe\scode\sa\slittle\ssmaller. -D 2014-09-26T02:41:05.726 +C Add\san\sassert()\sto\sverify\sthe\slast-row-id\sfor\sthe\sdatabase\sjust\sprior\sto\scalling\sa\sSQL\sfunction. +D 2014-09-26T18:30:11.093 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 91b7e12bca7b6056574ce28935e3e3f4769ce3c4 +F src/vdbe.c 93eeb6f9c3a3084133225a196f220454d71cca10 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h bb7f7ecfdead1a2ae0251b59f86f5724838d975c F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e93aecc090c2a1d3c231bb2bde044886eff0bdf7 -R 32be19747fff5e8f2465eed0f224b45d -U drh -Z f56b9000203c19d0f3a8172e8374b279 +P 83913515830aa850f9e38406f9422d7e88dcab66 +R dc4da31df9102dea4f18af6519657b79 +U mistachkin +Z 6fd91dd1e10822b3b57f7222d98e1fa9 diff --git a/manifest.uuid b/manifest.uuid index b182c8c316..482b361956 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83913515830aa850f9e38406f9422d7e88dcab66 \ No newline at end of file +d026f0c944ce812732d3595eaa3c5d432a86c7dd \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 1a1e441cd4..34eb1d42c5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1558,6 +1558,7 @@ case OP_Function: { ctx.pVdbe = p; MemSetTypeFlag(ctx.pOut, MEM_Null); ctx.fErrorOrAux = 0; + assert( db->lastRowid==lastRowid ); (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */