From facf47a891b114635ef5a5c2ea76ba1f8981e1c6 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Oct 2014 20:12:47 +0000 Subject: [PATCH 01/17] Use the padding word in the Mem object as temporary storage for serial_type value in OP_Record, and thus avoid a redundant computation of the serial_type for each column. FossilOrigin-Name: 4b3b65ee5ea61e9b9671ca027940bf02689cb890 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 4 ++-- src/vdbeInt.h | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f76c05d5cd..cdda42bff1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Three\ssmall\soptimizations\sto\svdbeaux.c. -D 2014-10-13T17:42:05.891 +C Use\sthe\spadding\sword\sin\sthe\sMem\sobject\sas\stemporary\sstorage\sfor\nserial_type\svalue\sin\sOP_Record,\sand\sthus\savoid\sa\sredundant\scomputation\sof\nthe\sserial_type\sfor\seach\scolumn. +D 2014-10-13T20:12:47.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,9 +289,9 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c cf7c24e531aad047689e4235a66b22526e765e46 +F src/vdbe.c 8755e3bb0d7d26b2b156c6f29ddd6b3d32b77df2 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h 90c0ae53382d16a4f77ad5a9c4b3e35278e9e061 +F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c2799aece17d347c64217a0e407bb10e50c184a3 -R f9a2ff50dcb19a8595444c5c4cad6f5f +P 04892f8ba6c55cec4fe37bfe59b6349fd2a40698 +R b0e17ef60b5aa0edd0fe18cf5fe3de58 U drh -Z bf4e3c223808aa3476302aafe1aa8090 +Z 6462e46ba313ee1bcf094c3a26197f24 diff --git a/manifest.uuid b/manifest.uuid index 02b54f919a..8d0a012b7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04892f8ba6c55cec4fe37bfe59b6349fd2a40698 \ No newline at end of file +4b3b65ee5ea61e9b9671ca027940bf02689cb890 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 7e0e35fb0e..049bf32ecd 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2616,7 +2616,7 @@ case OP_MakeRecord: { pRec = pLast; do{ assert( memIsValid(pRec) ); - serial_type = sqlite3VdbeSerialType(pRec, file_format); + pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format); len = sqlite3VdbeSerialTypeLen(serial_type); if( pRec->flags & MEM_Zero ){ if( nData ){ @@ -2665,7 +2665,7 @@ case OP_MakeRecord: { assert( pData0<=pLast ); pRec = pData0; do{ - serial_type = sqlite3VdbeSerialType(pRec, file_format); + serial_type = pRec->uTemp; i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ }while( (++pRec)<=pLast ); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 1b27e88e72..bb504d64a1 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -175,7 +175,7 @@ struct Mem { /* ShallowCopy only needs to copy the information above */ char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */ int szMalloc; /* Size of the zMalloc allocation */ - int iPadding1; /* Padding for 8-byte alignment */ + u32 uTemp; /* Transient storage for serial_type in OP_MakeRecord */ sqlite3 *db; /* The associated database connection */ void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG From 8dd8362d6446e8d83fd621122987f048216197c2 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Oct 2014 23:39:02 +0000 Subject: [PATCH 02/17] Optimize a database corruption test inside of the OP_Column opcode. FossilOrigin-Name: 005e5b388a8a97bca6d1f0e06c40d68d92aa1212 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 15 ++++++++------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index cdda42bff1..83cd0aa0f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sthe\spadding\sword\sin\sthe\sMem\sobject\sas\stemporary\sstorage\sfor\nserial_type\svalue\sin\sOP_Record,\sand\sthus\savoid\sa\sredundant\scomputation\sof\nthe\sserial_type\sfor\seach\scolumn. -D 2014-10-13T20:12:47.457 +C Optimize\sa\sdatabase\scorruption\stest\sinside\sof\sthe\sOP_Column\sopcode. +D 2014-10-13T23:39:02.463 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 8755e3bb0d7d26b2b156c6f29ddd6b3d32b77df2 +F src/vdbe.c 5ee15a66ce07e0482b92aa29e4dd0c5827a22d79 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 04892f8ba6c55cec4fe37bfe59b6349fd2a40698 -R b0e17ef60b5aa0edd0fe18cf5fe3de58 +P 4b3b65ee5ea61e9b9671ca027940bf02689cb890 +R 42e26b1c48be06a100ad5530a5448c12 U drh -Z 6462e46ba313ee1bcf094c3a26197f24 +Z 6b074ae9a762ef61cbe684b68fc815ba diff --git a/manifest.uuid b/manifest.uuid index 8d0a012b7d..1929c02aaf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b3b65ee5ea61e9b9671ca027940bf02689cb890 \ No newline at end of file +005e5b388a8a97bca6d1f0e06c40d68d92aa1212 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 049bf32ecd..0f9f45c456 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2417,15 +2417,16 @@ case OP_Column: { sMem.flags = MEM_Null; } - /* If we have read more header data than was contained in the header, - ** or if the end of the last field appears to be past the end of the - ** record, or if the end of the last field appears to be before the end - ** of the record (when all fields present), then we must be dealing - ** with a corrupt database. + /* The record is corrupt if any of the following are true: + ** (1) the bytes of the header extend past the declared header size + ** (zHdr>zEndHdr) + ** (2) the entire header was used but not all data was used + ** (zHdr==zEndHdr && offset!=pC->payloadSize) + ** (3) the end of the data extends beyond the end of the record. + ** (offset > pC->payloadSize) */ - if( (zHdr > zEndHdr) + if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset!=pC->payloadSize)) || (offset > pC->payloadSize) - || (zHdr==zEndHdr && offset!=pC->payloadSize) ){ rc = SQLITE_CORRUPT_BKPT; goto op_column_error; From 5604cc0d3b12e306cf7a601444e8d0f655e3bafc Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Oct 2014 20:25:43 +0000 Subject: [PATCH 03/17] Make sure new sqlite3_vtab objects created by the xCreate() virtual table method are initialized by the system, in accordance with the documentation. FossilOrigin-Name: eab82330631187dcc3e5d2dddd23dbda5752904b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vtab.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 83cd0aa0f1..94c543c920 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\sa\sdatabase\scorruption\stest\sinside\sof\sthe\sOP_Column\sopcode. -D 2014-10-13T23:39:02.463 +C Make\ssure\snew\ssqlite3_vtab\sobjects\screated\sby\sthe\sxCreate()\svirtual\stable\nmethod\sare\sinitialized\sby\sthe\ssystem,\sin\saccordance\swith\sthe\sdocumentation. +D 2014-10-14T20:25:43.215 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -298,7 +298,7 @@ F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 481327f50d9da330053aa7456702ce46d0a4e70f F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 -F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f +F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4b3b65ee5ea61e9b9671ca027940bf02689cb890 -R 42e26b1c48be06a100ad5530a5448c12 +P 005e5b388a8a97bca6d1f0e06c40d68d92aa1212 +R 69c5ef77aff899ac36ba1309415e232b U drh -Z 6b074ae9a762ef61cbe684b68fc815ba +Z ba859354cbc6c4d678eb09a46ba9f63b diff --git a/manifest.uuid b/manifest.uuid index 1929c02aaf..83a00ae67f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -005e5b388a8a97bca6d1f0e06c40d68d92aa1212 \ No newline at end of file +eab82330631187dcc3e5d2dddd23dbda5752904b \ No newline at end of file diff --git a/src/vtab.c b/src/vtab.c index c7a8a5a33f..faee4ae478 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -519,6 +519,7 @@ static int vtabCallConstructor( }else if( ALWAYS(pVTable->pVtab) ){ /* Justification of ALWAYS(): A correct vtab constructor must allocate ** the sqlite3_vtab object if successful. */ + memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); pVTable->pVtab->pModule = pMod->pModule; pVTable->nRef = 1; if( sCtx.pTab ){ From 74439167cb330f41e288270eba26b05a4f4af6a9 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Oct 2014 11:31:35 +0000 Subject: [PATCH 04/17] Rearrange an expression in vdbemem.c to avoid a (harmless) reference to a possibly unitialized variable. FossilOrigin-Name: 4a7b3fa049a9aa0668e318287edd4a78c0588bf8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbemem.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 94c543c920..c9b3f13dde 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\snew\ssqlite3_vtab\sobjects\screated\sby\sthe\sxCreate()\svirtual\stable\nmethod\sare\sinitialized\sby\sthe\ssystem,\sin\saccordance\swith\sthe\sdocumentation. -D 2014-10-14T20:25:43.215 +C Rearrange\san\sexpression\sin\svdbemem.c\sto\savoid\sa\s(harmless)\sreference\sto\sa\spossibly\sunitialized\svariable. +D 2014-10-15T11:31:35.560 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -295,7 +295,7 @@ F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 481327f50d9da330053aa7456702ce46d0a4e70f +F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 005e5b388a8a97bca6d1f0e06c40d68d92aa1212 -R 69c5ef77aff899ac36ba1309415e232b -U drh -Z ba859354cbc6c4d678eb09a46ba9f63b +P eab82330631187dcc3e5d2dddd23dbda5752904b +R c46357701a0c6687b1cdb44cc5cea3cc +U dan +Z f35b20ba175f90e21204e2589425beed diff --git a/manifest.uuid b/manifest.uuid index 83a00ae67f..2ed282008d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eab82330631187dcc3e5d2dddd23dbda5752904b \ No newline at end of file +4a7b3fa049a9aa0668e318287edd4a78c0588bf8 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 0c62db0720..870fb5bd89 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -143,7 +143,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ } } - if( pMem->z && bPreserve && pMem->z!=pMem->zMalloc ){ + if( bPreserve && pMem->z && pMem->z!=pMem->zMalloc ){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } if( (pMem->flags&MEM_Dyn)!=0 ){ From 92787cf0424ab04692843582c2ab9552c82b1c21 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Oct 2014 11:55:51 +0000 Subject: [PATCH 05/17] Add a four-byte prefix to the BtShared.pTmpSpace buffer to avoid reading before the beginning of an allocation. FossilOrigin-Name: 9386bfca128023583a24303e5f1d832987a49d43 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 22 +++++++++++++++++----- src/btreeInt.h | 2 +- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c9b3f13dde..66f8f35d8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rearrange\san\sexpression\sin\svdbemem.c\sto\savoid\sa\s(harmless)\sreference\sto\sa\spossibly\sunitialized\svariable. -D 2014-10-15T11:31:35.560 +C Add\sa\sfour-byte\sprefix\sto\sthe\sBtShared.pTmpSpace\sbuffer\sto\savoid\sreading\nbefore\sthe\sbeginning\sof\san\sallocation. +D 2014-10-15T11:55:51.434 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,9 +172,9 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c c9fcae8145436f728c61272cba72b1469c07f30d +F src/btree.c 1b1123cba0c65caa0baa51e71b8c089e3167c3ed F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 -F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d +F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eab82330631187dcc3e5d2dddd23dbda5752904b -R c46357701a0c6687b1cdb44cc5cea3cc -U dan -Z f35b20ba175f90e21204e2589425beed +P 4a7b3fa049a9aa0668e318287edd4a78c0588bf8 +R 0eaa9fa075886c1412c89d2319c66fd1 +U drh +Z fdb81bbd00d8de95e4a04ef38ee80f55 diff --git a/manifest.uuid b/manifest.uuid index 2ed282008d..864a483132 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a7b3fa049a9aa0668e318287edd4a78c0588bf8 \ No newline at end of file +9386bfca128023583a24303e5f1d832987a49d43 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 3553924c0f..758dfe6335 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2108,7 +2108,8 @@ static int removeFromSharingList(BtShared *pBt){ /* ** Make sure pBt->pTmpSpace points to an allocation of -** MX_CELL_SIZE(pBt) bytes. +** MX_CELL_SIZE(pBt) bytes with a 4-byte prefix for a left-child +** pointer. */ static void allocateTempSpace(BtShared *pBt){ if( !pBt->pTmpSpace ){ @@ -2123,8 +2124,16 @@ static void allocateTempSpace(BtShared *pBt){ ** it into a database page. This is not actually a problem, but it ** does cause a valgrind error when the 1 or 2 bytes of unitialized ** data is passed to system call write(). So to avoid this error, - ** zero the first 4 bytes of temp space here. */ - if( pBt->pTmpSpace ) memset(pBt->pTmpSpace, 0, 4); + ** zero the first 4 bytes of temp space here. + ** + ** Also: Provide four bytes of initialized space before the + ** beginning of pTmpSpace as an area available to prepend the + ** left-child pointer to the beginning of a cell. + */ + if( pBt->pTmpSpace ){ + memset(pBt->pTmpSpace, 0, 8); + pBt->pTmpSpace += 4; + } } } @@ -2132,8 +2141,11 @@ static void allocateTempSpace(BtShared *pBt){ ** Free the pBt->pTmpSpace allocation */ static void freeTempSpace(BtShared *pBt){ - sqlite3PageFree( pBt->pTmpSpace); - pBt->pTmpSpace = 0; + if( pBt->pTmpSpace ){ + pBt->pTmpSpace -= 4; + sqlite3PageFree(pBt->pTmpSpace); + pBt->pTmpSpace = 0; + } } /* diff --git a/src/btreeInt.h b/src/btreeInt.h index 9f648fceb0..2368e6c884 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -436,7 +436,7 @@ struct BtShared { BtLock *pLock; /* List of locks held on this shared-btree struct */ Btree *pWriter; /* Btree with currently open write transaction */ #endif - u8 *pTmpSpace; /* BtShared.pageSize bytes of space for tmp use */ + u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ }; /* From f41652419e75b83f7ebbcce40bffb5983087eb18 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Oct 2014 14:45:34 +0000 Subject: [PATCH 06/17] Fix a problem causing lock5.test to fail in mmap-mode. FossilOrigin-Name: b3e7b446bdb47cf9d7fe43dc37e3b4f8010ee09e --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/lock5.test | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 66f8f35d8a..479beb42cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sfour-byte\sprefix\sto\sthe\sBtShared.pTmpSpace\sbuffer\sto\savoid\sreading\nbefore\sthe\sbeginning\sof\san\sallocation. -D 2014-10-15T11:55:51.434 +C Fix\sa\sproblem\scausing\slock5.test\sto\sfail\sin\smmap-mode. +D 2014-10-15T14:45:34.650 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -677,7 +677,7 @@ F test/lock.test 87af515b0c4cf928576d0f89946d67d7c265dfb4 F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12 -F test/lock5.test 5ad6a1f536036ff1be915cfdd41481aeafda3273 +F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431 F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4a7b3fa049a9aa0668e318287edd4a78c0588bf8 -R 0eaa9fa075886c1412c89d2319c66fd1 -U drh -Z fdb81bbd00d8de95e4a04ef38ee80f55 +P 9386bfca128023583a24303e5f1d832987a49d43 +R d767fb05b1c758bd73ecd7ed19e0f8f8 +U dan +Z deb18f7b499c6ed23800c00a8fad84b5 diff --git a/manifest.uuid b/manifest.uuid index 864a483132..ff3f963720 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9386bfca128023583a24303e5f1d832987a49d43 \ No newline at end of file +b3e7b446bdb47cf9d7fe43dc37e3b4f8010ee09e \ No newline at end of file diff --git a/test/lock5.test b/test/lock5.test index f0d495508a..99214afb19 100644 --- a/test/lock5.test +++ b/test/lock5.test @@ -154,6 +154,7 @@ do_test lock5-flock.8 { do_test lock5-none.1 { sqlite3 db test.db -vfs unix-none sqlite3 db2 test.db -vfs unix-none + execsql { PRAGMA mmap_size = 0 } db2 execsql { BEGIN; INSERT INTO t1 VALUES(3, 4); @@ -162,8 +163,8 @@ do_test lock5-none.1 { do_test lock5-none.2 { execsql { SELECT * FROM t1 } } {1 2 3 4} -do_test lock5-flock.3 { - execsql { SELECT * FROM t1 } db2 +do_test lock5-none.3 { + execsql { SELECT * FROM t1; } db2 } {1 2} do_test lock5-none.4 { execsql { @@ -183,7 +184,7 @@ ifcapable memorymanage { } {1 2 3 4} } -do_test lock5-flock.X { +do_test lock5-none.X { db close db2 close } {} From 5d510d4c412ec360cec5cd088fd8debff94e3354 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Oct 2014 15:28:27 +0000 Subject: [PATCH 07/17] Update releasetest.tcl so that x86-64 runs a superset of the x86 tests. FossilOrigin-Name: 3c1e70f4d55bc009ed9ed4cf6d756d7061985851 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/releasetest.tcl | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 479beb42cc..d416b82e23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\slock5.test\sto\sfail\sin\smmap-mode. -D 2014-10-15T14:45:34.650 +C Update\sreleasetest.tcl\sso\sthat\sx86-64\sruns\sa\ssuperset\sof\sthe\sx86\stests. +D 2014-10-15T15:28:27.041 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -783,7 +783,7 @@ F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a -F test/releasetest.tcl 4296b9adbc5992bcd0b0b2876b7651f57c1494f2 +F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9386bfca128023583a24303e5f1d832987a49d43 -R d767fb05b1c758bd73ecd7ed19e0f8f8 +P b3e7b446bdb47cf9d7fe43dc37e3b4f8010ee09e +R d10aa2fd16b5624d36b7172d8913c506 U dan -Z deb18f7b499c6ed23800c00a8fad84b5 +Z 50710860f359ce0bbdbe754b3e261318 diff --git a/manifest.uuid b/manifest.uuid index ff3f963720..5726fd827b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b3e7b446bdb47cf9d7fe43dc37e3b4f8010ee09e \ No newline at end of file +3c1e70f4d55bc009ed9ed4cf6d756d7061985851 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 589319d680..d2a1bd2bb0 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -196,7 +196,8 @@ array set ::Platforms { "Device-Two" test "Ftrapv" test "No-lookaside" test - "Default" "threadtest test" + "Devkit" test + "Default" "threadtest fulltest" "Device-One" fulltest } Linux-i686 { From facd5fd6221a547a1fd92c0ee517d3978f34ce6d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Oct 2014 11:45:14 +0000 Subject: [PATCH 08/17] Update the vdbe-compress.tcl script to account for expressions of the form (123>var), where "var" should be replaced with a reference to a union member. FossilOrigin-Name: 640345d880c6178f8434e3ce40329b7527588843 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/vdbe-compress.tcl | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d416b82e23..603373ecda 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sreleasetest.tcl\sso\sthat\sx86-64\sruns\sa\ssuperset\sof\sthe\sx86\stests. -D 2014-10-15T15:28:27.041 +C Update\sthe\svdbe-compress.tcl\sscript\sto\saccount\sfor\sexpressions\sof\sthe\sform\s(123>var),\swhere\s"var"\sshould\sbe\sreplaced\swith\sa\sreference\sto\sa\sunion\smember. +D 2014-10-16T11:45:14.341 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1199,12 +1199,12 @@ F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 -F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 +F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b3e7b446bdb47cf9d7fe43dc37e3b4f8010ee09e -R d10aa2fd16b5624d36b7172d8913c506 +P 3c1e70f4d55bc009ed9ed4cf6d756d7061985851 +R e0f61b3d5f668f3cfe77a35e5da93d93 U dan -Z 50710860f359ce0bbdbe754b3e261318 +Z d051f6e18c5411d016dc382904d39726 diff --git a/manifest.uuid b/manifest.uuid index 5726fd827b..be86e66b80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c1e70f4d55bc009ed9ed4cf6d756d7061985851 \ No newline at end of file +640345d880c6178f8434e3ce40329b7527588843 \ No newline at end of file diff --git a/tool/vdbe-compress.tcl b/tool/vdbe-compress.tcl index a349830bcf..9477f4afe6 100644 --- a/tool/vdbe-compress.tcl +++ b/tool/vdbe-compress.tcl @@ -110,6 +110,11 @@ while {![eof stdin]} { foreach v $vlist { regsub -all "(\[^a-zA-Z0-9>.\])${v}(\\W)" $line "\\1u.$sname.$v\\2" line regsub -all "(\[^a-zA-Z0-9>.\])${v}(\\W)" $line "\\1u.$sname.$v\\2" line + + # The expressions above fail to catch instance of variable "abc" in + # expressions like (32>abc). The following expression makes those + # substitutions. + regsub -all "(\[^-\])>${v}(\\W)" $line "\\1>u.$sname.$v\\2" line } append afterUnion [string trimright $line]\n } elseif {$line=="" && [eof stdin]} { From 92c2e0da8f6fbec24e953f8b1acf26c6d33f55e0 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 16 Oct 2014 18:34:50 +0000 Subject: [PATCH 09/17] Changes to work around Win32 and MSVCRT APIs that are not present on Windows CE 200x. FossilOrigin-Name: 1418c006e377d7915a50577d4ccb21125b750bae --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_win.c | 7 ++++++- src/threads.c | 4 ++-- test/sort.test | 1 + 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 603373ecda..fb97e6165f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\svdbe-compress.tcl\sscript\sto\saccount\sfor\sexpressions\sof\sthe\sform\s(123>var),\swhere\s"var"\sshould\sbe\sreplaced\swith\sa\sreference\sto\sa\sunion\smember. -D 2014-10-16T11:45:14.341 +C Changes\sto\swork\saround\sWin32\sand\sMSVCRT\sAPIs\sthat\sare\snot\spresent\son\sWindows\sCE\s200x. +D 2014-10-16T18:34:50.284 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 -F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 +F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c a171cf9dd09c6cb162b262c328d4dfd198e04f80 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -282,7 +282,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 60c9d400abf17ccdc8767cdc6af90b9c5acf58bd +F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 @@ -846,7 +846,7 @@ F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 15e1d3014abc3f6d4357ed81b93b82117aefd235 +F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d F test/sort2.test 269f4f50c6e468cc32b302ae7ff0add8338ec6de F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 F test/sort4.test 6c37d85f7cd28d50cce222fcab84ccd771e105cb @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3c1e70f4d55bc009ed9ed4cf6d756d7061985851 -R e0f61b3d5f668f3cfe77a35e5da93d93 -U dan -Z d051f6e18c5411d016dc382904d39726 +P 640345d880c6178f8434e3ce40329b7527588843 +R 46582f87dc5791f77f8a445cb47feb38 +U mistachkin +Z 04c8b48d78aac4913dca1bde0b860300 diff --git a/manifest.uuid b/manifest.uuid index be86e66b80..2cab3291b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -640345d880c6178f8434e3ce40329b7527588843 \ No newline at end of file +1418c006e377d7915a50577d4ccb21125b750bae \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index e12ce4e532..8ca2107d90 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -943,7 +943,11 @@ static struct win_syscall { #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ DWORD))aSyscall[63].pCurrent) +#if !SQLITE_OS_WINCE { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, +#else + { "WaitForSingleObjectEx", (SYSCALL)0, 0 }, +#endif #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ BOOL))aSyscall[64].pCurrent) @@ -1286,7 +1290,8 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #endif } -#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ + SQLITE_THREADSAFE>0 DWORD sqlite3Win32Wait(HANDLE hObject){ DWORD rc; while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, diff --git a/src/threads.c b/src/threads.c index 6d39042fd9..18d7320a12 100644 --- a/src/threads.c +++ b/src/threads.c @@ -98,7 +98,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /********************************* Win32 Threads ****************************/ -#if SQLITE_OS_WIN && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 +#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 #define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ #include @@ -191,7 +191,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; } -#endif /* SQLITE_OS_WIN && !SQLITE_OS_WINRT */ +#endif /* SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT */ /******************************** End Win32 Threads *************************/ diff --git a/test/sort.test b/test/sort.test index 1c89552bb1..be2a6f531c 100644 --- a/test/sort.test +++ b/test/sort.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix sort # Create a bunch of data to sort against # From 7bdc9749d5f3d34fe7ba6051673a58229f8fe91d Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 16 Oct 2014 21:39:17 +0000 Subject: [PATCH 10/17] Work around MSVC not being able to deduce that a local variable was initialized in a called function. FossilOrigin-Name: 06c576c152c4013080c255cbbeb45bf2e298be9f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fb97e6165f..6426f0698c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\swork\saround\sWin32\sand\sMSVCRT\sAPIs\sthat\sare\snot\spresent\son\sWindows\sCE\s200x. -D 2014-10-16T18:34:50.284 +C Work\saround\sMSVC\snot\sbeing\sable\sto\sdeduce\sthat\sa\slocal\svariable\swas\sinitialized\sin\sa\scalled\sfunction. +D 2014-10-16T21:39:17.263 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef +F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 640345d880c6178f8434e3ce40329b7527588843 -R 46582f87dc5791f77f8a445cb47feb38 +P 1418c006e377d7915a50577d4ccb21125b750bae +R 76bb09f8a8f8bdebeb708f5bb39aa31b U mistachkin -Z 04c8b48d78aac4913dca1bde0b860300 +Z baca4535ffaf8e732b4d2d60c4d0a012 diff --git a/manifest.uuid b/manifest.uuid index 2cab3291b8..5f80a79efb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1418c006e377d7915a50577d4ccb21125b750bae \ No newline at end of file +06c576c152c4013080c255cbbeb45bf2e298be9f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index d9679caa06..46c9f3789d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -2292,7 +2292,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ assert( pSorter->bUseThreads==0 || pSorter->nTask>1 ); if( pSorter->bUseThreads ){ int iTask; - PmaReader *pReadr; + PmaReader *pReadr = 0; SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; rc = vdbeSortAllocUnpacked(pLast); if( rc==SQLITE_OK ){ From 81b567a4c73c06069d72b95ee2b09a0aca46b3ec Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Oct 2014 11:24:17 +0000 Subject: [PATCH 11/17] Version 3.8.7 FossilOrigin-Name: e4ab094f8afce0817f4074e823fabe59fc29ebb4 --- manifest | 13 ++++++++----- manifest.uuid | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/manifest b/manifest index 6426f0698c..e210943d15 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\sMSVC\snot\sbeing\sable\sto\sdeduce\sthat\sa\slocal\svariable\swas\sinitialized\sin\sa\scalled\sfunction. -D 2014-10-16T21:39:17.263 +C Version\s3.8.7 +D 2014-10-17T11:24:17.839 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1204,7 +1204,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1418c006e377d7915a50577d4ccb21125b750bae +P 06c576c152c4013080c255cbbeb45bf2e298be9f R 76bb09f8a8f8bdebeb708f5bb39aa31b -U mistachkin -Z baca4535ffaf8e732b4d2d60c4d0a012 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.8.7 * +U drh +Z b2f516d1147acb0e1bf1c700327ee52e diff --git a/manifest.uuid b/manifest.uuid index 5f80a79efb..f4f611fe44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06c576c152c4013080c255cbbeb45bf2e298be9f \ No newline at end of file +e4ab094f8afce0817f4074e823fabe59fc29ebb4 \ No newline at end of file From 40253262e0b3f9bfd79b6a57c891eb0f0459a38f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Oct 2014 21:35:05 +0000 Subject: [PATCH 12/17] Fix a (probably harmless) but in the CSV output mode of the command-line shell. FossilOrigin-Name: 19fe4a0a475bd94f491031aea7a183f7c0515cf3 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/shell.c | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e210943d15..894c7cd51a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.7 -D 2014-10-17T11:24:17.839 +C Fix\sa\s(probably\sharmless)\sbut\sin\sthe\sCSV\soutput\smode\sof\sthe\scommand-line\nshell. +D 2014-10-17T21:35:05.954 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 18ee8bbe9502d8848072dc2eddd1ea09254ba494 +F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1204,10 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 06c576c152c4013080c255cbbeb45bf2e298be9f -R 76bb09f8a8f8bdebeb708f5bb39aa31b -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.7 * +P e4ab094f8afce0817f4074e823fabe59fc29ebb4 +R bacb142e035c538b5014a902fee951f4 U drh -Z b2f516d1147acb0e1bf1c700327ee52e +Z 99accb7d1ae416e10fed302750dd8f99 diff --git a/manifest.uuid b/manifest.uuid index f4f611fe44..db419ccbec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4ab094f8afce0817f4074e823fabe59fc29ebb4 \ No newline at end of file +19fe4a0a475bd94f491031aea7a183f7c0515cf3 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 3ca4b094bc..59cd2011e7 100644 --- a/src/shell.c +++ b/src/shell.c @@ -882,7 +882,7 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int } fprintf(p->out,"%s",p->newline); } - if( azArg>0 ){ + if( nArg>0 ){ for(i=0; i Date: Tue, 21 Oct 2014 01:05:09 +0000 Subject: [PATCH 13/17] If a skip-scan is a proper subset of some other scan, then adjust the cost of the skip-scan upward so that it is more costly than the other scan. Such a cost imbalance can arise under STAT4 because of difficulties in getting an accurate estimate for skip-scans. FossilOrigin-Name: f4b22a2620a5dc48949048c2ecbd226755d4b2c3 --- manifest | 15 ++--- manifest.uuid | 2 +- src/where.c | 121 +++++++++++++++++------------------- src/whereInt.h | 4 +- test/skipscan6.test | 145 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 212 insertions(+), 75 deletions(-) create mode 100644 test/skipscan6.test diff --git a/manifest b/manifest index 894c7cd51a..3c0097dfe9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\s(probably\sharmless)\sbut\sin\sthe\sCSV\soutput\smode\sof\sthe\scommand-line\nshell. -D 2014-10-17T21:35:05.954 +C If\sa\sskip-scan\sis\sa\sproper\ssubset\sof\ssome\sother\sscan,\sthen\sadjust\sthe\ncost\sof\sthe\sskip-scan\supward\sso\sthat\sit\sis\smore\scostly\sthan\sthe\sother\sscan.\nSuch\sa\scost\simbalance\scan\sarise\sunder\sSTAT4\sbecause\sof\sdifficulties\sin\sgetting\nan\saccurate\sestimate\sfor\sskip-scans. +D 2014-10-21T01:05:09.795 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,8 +302,8 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 -F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c +F src/where.c 2d1ff51eede0e4dcc87569dc8e3161237295162a +F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -844,6 +844,7 @@ F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 +F test/skipscan6.test 3a891b45d6df266ced861a2ad9d03fca2bc7fcc5 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d @@ -1204,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e4ab094f8afce0817f4074e823fabe59fc29ebb4 -R bacb142e035c538b5014a902fee951f4 +P 19fe4a0a475bd94f491031aea7a183f7c0515cf3 +R 7ae877aeccb485915279368cd2f54415 U drh -Z 99accb7d1ae416e10fed302750dd8f99 +Z dece11d7f7de853e09f0182395012c8f diff --git a/manifest.uuid b/manifest.uuid index db419ccbec..ee9b76f602 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19fe4a0a475bd94f491031aea7a183f7c0515cf3 \ No newline at end of file +f4b22a2620a5dc48949048c2ecbd226755d4b2c3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index bc0110779e..00b1880880 100644 --- a/src/where.c +++ b/src/where.c @@ -2638,7 +2638,7 @@ static int codeAllEqualityTerms( pLoop = pLevel->pWLoop; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); nEq = pLoop->u.btree.nEq; - nSkip = pLoop->u.btree.nSkip; + nSkip = pLoop->nSkip; pIdx = pLoop->u.btree.pIndex; assert( pIdx!=0 ); @@ -2752,7 +2752,7 @@ static void explainAppendTerm( static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ Index *pIndex = pLoop->u.btree.pIndex; u16 nEq = pLoop->u.btree.nEq; - u16 nSkip = pLoop->u.btree.nSkip; + u16 nSkip = pLoop->nSkip; int i, j; Column *aCol = pTab->aCol; i16 *aiColumn = pIndex->aiColumn; @@ -3189,7 +3189,7 @@ static Bitmask codeOneLoopStart( pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; - assert( nEq>=pLoop->u.btree.nSkip ); + assert( nEq>=pLoop->nSkip ); /* If this loop satisfies a sort order (pOrderBy) request that ** was passed to this function to implement a "SELECT min(x) ..." @@ -3206,7 +3206,7 @@ static Bitmask codeOneLoopStart( && pWInfo->nOBSat>0 && (pIdx->nKeyCol>nEq) ){ - assert( pLoop->u.btree.nSkip==0 ); + assert( pLoop->nSkip==0 ); bSeekPastNull = 1; nExtraReg = 1; } @@ -3827,7 +3827,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3_free(z); } if( p->wsFlags & WHERE_SKIPSCAN ){ - sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->u.btree.nSkip); + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); }else{ sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); } @@ -3956,12 +3956,15 @@ static int whereLoopCheaperProperSubset( const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; - if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* X is not a subset of Y */ + } if( pX->rRun >= pY->rRun ){ if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ } for(i=pX->nLTerm-1; i>=0; i--){ + if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } @@ -3983,25 +3986,12 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. -** -** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the -** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE -** clause terms covered, since some of the first nLTerm entries in aLTerm[] -** will be NULL (because they are skipped). That makes it more difficult -** to compare the loops. We could add extra code to do the comparison, and -** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this -** adjustment is sufficient minor, that it is very difficult to construct -** a test case where the extra code would improve the query plan. Better -** to avoid the added complexity and just omit cost adjustments to SKIPSCAN -** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; - if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ @@ -4295,7 +4285,7 @@ static int whereLoopAddBtreeIndex( Bitmask saved_prereq; /* Original value of pNew->prereq */ u16 saved_nLTerm; /* Original value of pNew->nLTerm */ u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ - u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ + u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ @@ -4324,7 +4314,7 @@ static int whereLoopAddBtreeIndex( pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); saved_nEq = pNew->u.btree.nEq; - saved_nSkip = pNew->u.btree.nSkip; + saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; @@ -4332,44 +4322,6 @@ static int whereLoopAddBtreeIndex( pNew->rSetup = 0; rSize = pProbe->aiRowLogEst[0]; rLogSize = estLog(rSize); - - /* Consider using a skip-scan if there are no WHERE clause constraints - ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. - ** - ** The magic number 18 is selected on the basis that scanning 17 rows - ** is almost always quicker than an index seek (even though if the index - ** contains fewer than 2^17 rows we assume otherwise in other parts of - ** the code). And, even if it is not, it should not be too much slower. - ** On the other hand, the extra seeks could end up being significantly - ** more expensive. */ - assert( 42==sqlite3LogEst(18) ); - if( saved_nEq==saved_nSkip - && saved_nEq+1nKeyCol - && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ - && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK - ){ - LogEst nIter; - pNew->u.btree.nEq++; - pNew->u.btree.nSkip++; - pNew->aLTerm[pNew->nLTerm++] = 0; - pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - if( pTerm ){ - /* TUNING: When estimating skip-scan for a term that is also indexable, - ** multiply the cost of the skip-scan by 2.0, to make it a little less - ** desirable than the regular index lookup. */ - nIter += 10; assert( 10==sqlite3LogEst(2) ); - } - pNew->nOut -= nIter; - /* TUNING: Because uncertainties in the estimates for skip-scan queries, - ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ - nIter += 5; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); - pNew->nOut = saved_nOut; - pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; - } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; @@ -4532,10 +4484,50 @@ static int whereLoopAddBtreeIndex( } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; + pNew->nSkip = saved_nSkip; pNew->wsFlags = saved_wsFlags; pNew->nOut = saved_nOut; pNew->nLTerm = saved_nLTerm; + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + if( pTerm ){ + /* TUNING: When estimating skip-scan for a term that is also indexable, + ** multiply the cost of the skip-scan by 2.0, to make it a little less + ** desirable than the regular index lookup. */ + nIter += 10; assert( 10==sqlite3LogEst(2) ); + } + pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + } + return rc; } @@ -4714,7 +4706,7 @@ static int whereLoopAddBtree( if( pTerm->prereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; @@ -4755,7 +4747,7 @@ static int whereLoopAddBtree( } rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; pNew->rSetup = 0; @@ -5305,7 +5297,7 @@ static i8 wherePathSatisfiesOrderBy( /* Skip over == and IS NULL terms */ if( ju.btree.nEq - && pLoop->u.btree.nSkip==0 + && pLoop->nSkip==0 && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 ){ if( i & WO_ISNULL ){ @@ -5878,7 +5870,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; - pLoop->u.btree.nSkip = 0; + pLoop->nSkip = 0; pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); if( pTerm ){ pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; @@ -5890,7 +5882,6 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pLoop->aLTermSpace==pLoop->aLTerm ); - assert( ArraySize(pLoop->aLTermSpace)==4 ); if( !IsUniqueIndex(pIdx) || pIdx->pPartIdxWhere!=0 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) diff --git a/src/whereInt.h b/src/whereInt.h index f17906e63a..e9eb8b7dd2 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -115,7 +115,6 @@ struct WhereLoop { union { struct { /* Information for internal btree tables */ u16 nEq; /* Number of equality constraints */ - u16 nSkip; /* Number of initial index columns to skip */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ @@ -128,12 +127,13 @@ struct WhereLoop { } u; u32 wsFlags; /* WHERE_* flags describing the plan */ u16 nLTerm; /* Number of entries in aLTerm[] */ + u16 nSkip; /* Number of NULL aLTerm[] entries */ /**** whereLoopXfer() copies fields above ***********************/ # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) u16 nLSlot; /* Number of slots allocated for aLTerm[] */ WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ - WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ + WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ }; /* This object holds the prerequisites and the cost of running a diff --git a/test/skipscan6.test b/test/skipscan6.test new file mode 100644 index 0000000000..9eda9a66f3 --- /dev/null +++ b/test/skipscan6.test @@ -0,0 +1,145 @@ +# 2014-10-21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. In +# particular, this file verifies that use of all columns of an index +# is always preferred over the use of a skip-scan on some columns of +# the same index. Because of difficulties in scoring a skip-scan, +# the skip-scan can sometimes come out with a lower raw score when +# using STAT4. But the query planner should detect this and use the +# full index rather than the skip-scan. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix skipscan6 + +ifcapable !stat4 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE TABLE t1( + aa int, + bb int, + cc int, + dd int, + ee int + ); + CREATE INDEX ix on t1(aa, bb, cc, dd DESC); + ANALYZE sqlite_master; + INSERT INTO sqlite_stat1 VALUES('t1','ix','2695116 1347558 264 18 2'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35'); + ANALYZE sqlite_master; +} {} +do_execsql_test 1.2 { + EXPLAIN QUERY PLAN + SELECT COUNT(*) + FROM t1 + WHERE bb=21 + AND aa=1 + AND dd BETWEEN 1413833728 and 1413837331; +} {/INDEX ix .aa=. AND bb=../} + +do_execsql_test 2.1 { + DROP INDEX ix; + CREATE INDEX good on t1(bb, aa, dd DESC); + CREATE INDEX bad on t1(aa, bb, cc, dd DESC); + DELETE FROM sqlite_stat1; + DELETE FROM sqlite_stat4; + INSERT INTO sqlite_stat1 VALUES('t1','good','2695116 299 264 2'); + INSERT INTO sqlite_stat1 VALUES('t1','bad','2695116 1347558 264 18 2'); + INSERT INTO sqlite_stat4 VALUES('t1','good','197030 196859 32 1','15086 15086 92511 92536','19 25 81644 92536',X'05010904031552977BD725BD22'); + INSERT INTO sqlite_stat4 VALUES('t1','good','14972 14687 1 1','289878 289878 299457 299457','199 244 267460 299457',X'050209040301344F7E569402C419'); + INSERT INTO sqlite_stat4 VALUES('t1','good','19600 19313 22 1','327127 327127 346222 346243','261 319 306884 346243',X'0502090403018A49503BC01EC577'); + INSERT INTO sqlite_stat4 VALUES('t1','good','25666 25047 15 1','352087 352087 372692 372706','266 327 325601 372706',X'050209040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','good','42392 42327 26 1','378657 378657 382547 382572','268 331 333529 382572',X'05020904030193533B2FE326ED48'); + INSERT INTO sqlite_stat4 VALUES('t1','good','24619 24513 11 1','457872 457872 461748 461758','286 358 399322 461758',X'050209040301A752B1557825EA7C'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18969 18730 15 1','482491 482491 501105 501119','287 360 433605 501119',X'050209040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','good','119710 119603 1 1','576500 576500 598915 598915','404 505 519877 598915',X'05020904030247539A7A7912F617'); + INSERT INTO sqlite_stat4 VALUES('t1','good','11955 11946 1 1','889796 889796 898373 898373','938 1123 794694 898373',X'050209040304EF4DF9C4150BBB28'); + INSERT INTO sqlite_stat4 VALUES('t1','good','57197 57138 24 1','1129865 1129865 1151492 1151515','1967 2273 1027048 1151515',X'05020904030988533510BC26E20A'); + INSERT INTO sqlite_stat4 VALUES('t1','good','3609 3543 1 1','1196265 1196265 1197831 1197831','2002 2313 1070108 1197831',X'050209040309B050E95CD718D94D'); + INSERT INTO sqlite_stat4 VALUES('t1','good','25391 25365 13 1','1309378 1309378 1315567 1315579','2561 2936 1178358 1315579',X'05020904030C5F53DF9E13283570'); + INSERT INTO sqlite_stat4 VALUES('t1','good','45232 45180 17 1','1334769 1334769 1337946 1337962','2562 2938 1198998 1337962',X'05020904030C60541CACEE28BCAC'); + INSERT INTO sqlite_stat4 VALUES('t1','good','5496 5493 1 1','1495882 1495882 1497289 1497289','3043 3479 1348695 1497289',X'05020904030E99515C62AD0F0B34'); + INSERT INTO sqlite_stat4 VALUES('t1','good','26348 26139 17 1','1517381 1517381 1529990 1530006','3074 3519 1378320 1530006',X'05020904030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','good','102927 102894 10 1','1547088 1547088 1649950 1649959','3109 3559 1494260 1649959',X'05020904030EE34D309F671FFA47'); + INSERT INTO sqlite_stat4 VALUES('t1','good','3602 3576 1 1','1793873 1793873 1796747 1796747','3601 4128 1630783 1796747',X'050209040311294FE88B432219B9'); + INSERT INTO sqlite_stat4 VALUES('t1','good','154 154 1 1','2096059 2096059 2096205 2096205','5037 5779 1893039 2096205',X'050209040317994EFF05A016DCED'); + INSERT INTO sqlite_stat4 VALUES('t1','good','68153 66574 60 1','2244039 2244039 2268892 2268951','5899 6749 2027553 2268951',X'05020904031B8A532DBC5A26D2BA'); + INSERT INTO sqlite_stat4 VALUES('t1','good','321 321 1 1','2395618 2395618 2395663 2395663','6609 7528 2118435 2395663',X'05020904031EFA54078EEE1E2D65'); + INSERT INTO sqlite_stat4 VALUES('t1','good','19449 19440 22 1','2407769 2407769 2426049 2426070','6718 7651 2146904 2426070',X'05020904031F7450E6118C2336BD'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18383 18321 56 1','2539949 2539949 2551080 2551135','7838 8897 2245459 2551135',X'050209040324A752EA2E1E2642B2'); + INSERT INTO sqlite_stat4 VALUES('t1','good','22479 22384 60 1','2558332 2558332 2565233 2565292','7839 8899 2251202 2565292',X'050209040324A853926538279A5F'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18771 18699 63 1','2580811 2580811 2596914 2596976','7840 8901 2263572 2596976',X'050209040324A9526C1DE9256E72'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35'); + ANALYZE sqlite_master; +} {} +do_execsql_test 2.2 { + EXPLAIN QUERY PLAN + SELECT COUNT(*) + FROM t1 + WHERE bb=21 + AND aa=1 + AND dd BETWEEN 1413833728 and 1413837331; +} {/INDEX good .bb=. AND aa=. AND dd>. AND dd<../} + + + +finish_test From 1b131b7a7acc83c7bd1a5e7d0872c968d9e26489 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Oct 2014 16:01:40 +0000 Subject: [PATCH 14/17] Improvements to the WHERETRACE debugging logic. FossilOrigin-Name: ec1e942f08548695ff02645b3f3cd6bb2516bc9a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3c0097dfe9..d708dca4f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\sskip-scan\sis\sa\sproper\ssubset\sof\ssome\sother\sscan,\sthen\sadjust\sthe\ncost\sof\sthe\sskip-scan\supward\sso\sthat\sit\sis\smore\scostly\sthan\sthe\sother\sscan.\nSuch\sa\scost\simbalance\scan\sarise\sunder\sSTAT4\sbecause\sof\sdifficulties\sin\sgetting\nan\saccurate\sestimate\sfor\sskip-scans. -D 2014-10-21T01:05:09.795 +C Improvements\sto\sthe\sWHERETRACE\sdebugging\slogic. +D 2014-10-21T16:01:40.774 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2d1ff51eede0e4dcc87569dc8e3161237295162a +F src/where.c 45cb63cb1422d7e5a9229c297e978d294ae51e16 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 19fe4a0a475bd94f491031aea7a183f7c0515cf3 -R 7ae877aeccb485915279368cd2f54415 +P f4b22a2620a5dc48949048c2ecbd226755d4b2c3 +R 59b7c6284af5e31186eaa6edb85f9e59 U drh -Z dece11d7f7de853e09f0182395012c8f +Z 3eaff9a0b8f4f97b5dd119bc1e1acc8b diff --git a/manifest.uuid b/manifest.uuid index ee9b76f602..838729f698 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4b22a2620a5dc48949048c2ecbd226755d4b2c3 \ No newline at end of file +ec1e942f08548695ff02645b3f3cd6bb2516bc9a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 00b1880880..7dfe0f02bb 100644 --- a/src/where.c +++ b/src/where.c @@ -3995,11 +3995,15 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut - 1; }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ /* Adjust pTemplate cost upward so that it is costlier than p since ** pTemplate is a proper subset of p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut + 1; } @@ -5751,7 +5755,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } #ifdef WHERETRACE_ENABLED /* >=2 */ - if( sqlite3WhereTrace>=2 ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("---- after round %d ----\n", iLoop); for(ii=0, pTo=aTo; ii Date: Tue, 21 Oct 2014 18:16:21 +0000 Subject: [PATCH 15/17] Further tuning of the cost estimates for skip-scan loops, especially for cases when skip-scan loops are in competition with regular loops. FossilOrigin-Name: a27861c28c4791e51d797aa37e9cca806cb58775 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 10 +++------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index d708dca4f3..3daca49fdd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\sWHERETRACE\sdebugging\slogic. -D 2014-10-21T16:01:40.774 +C Further\stuning\sof\sthe\scost\sestimates\sfor\sskip-scan\sloops,\sespecially\sfor\scases\nwhen\sskip-scan\sloops\sare\sin\scompetition\swith\sregular\sloops. +D 2014-10-21T18:16:21.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 45cb63cb1422d7e5a9229c297e978d294ae51e16 +F src/where.c 994b38c8697aad095878ef1e4860902df457427f F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f4b22a2620a5dc48949048c2ecbd226755d4b2c3 -R 59b7c6284af5e31186eaa6edb85f9e59 +P ec1e942f08548695ff02645b3f3cd6bb2516bc9a +R 4f5d15d10f268e58d7c047d490bf8adc U drh -Z 3eaff9a0b8f4f97b5dd119bc1e1acc8b +Z 7d7f3f0271fe8b8df2863a2c22053442 diff --git a/manifest.uuid b/manifest.uuid index 838729f698..52d4aafd99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec1e942f08548695ff02645b3f3cd6bb2516bc9a \ No newline at end of file +a27861c28c4791e51d797aa37e9cca806cb58775 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7dfe0f02bb..7f51d00c87 100644 --- a/src/where.c +++ b/src/where.c @@ -3994,7 +3994,9 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p */ + ** subset p. Except, do not adjust the cost estimate downward for + ** a loop that skips more columns. */ + if( pTemplate->nSkip>p->nSkip ) continue; WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; @@ -4515,12 +4517,6 @@ static int whereLoopAddBtreeIndex( pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - if( pTerm ){ - /* TUNING: When estimating skip-scan for a term that is also indexable, - ** multiply the cost of the skip-scan by 2.0, to make it a little less - ** desirable than the regular index lookup. */ - nIter += 10; assert( 10==sqlite3LogEst(2) ); - } pNew->nOut -= nIter; /* TUNING: Because uncertainties in the estimates for skip-scan queries, ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ From 442c5cd3cfc67d3e10aa64d9f180ef94fb3597cf Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Oct 2014 21:56:06 +0000 Subject: [PATCH 16/17] Call fsync() right after ftruncate() when in journal_mode=TRUNCATE and when synchronous=FULL in order to ensure that transactions are durable across a power loss that happens moments after the commit. Proposed fix for [https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. FossilOrigin-Name: 3e922208b68563489c7766abb9afb4885113e7b8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3daca49fdd..f4a8a2a556 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stuning\sof\sthe\scost\sestimates\sfor\sskip-scan\sloops,\sespecially\sfor\scases\nwhen\sskip-scan\sloops\sare\sin\scompetition\swith\sregular\sloops. -D 2014-10-21T18:16:21.388 +C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. +D 2014-10-21T21:56:06.890 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c a171cf9dd09c6cb162b262c328d4dfd198e04f80 +F src/pager.c a98547ad9b1b5dbbc5e7d1c520be041b5d2c0926 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ec1e942f08548695ff02645b3f3cd6bb2516bc9a -R 4f5d15d10f268e58d7c047d490bf8adc +P a27861c28c4791e51d797aa37e9cca806cb58775 +R 1375151e695773da263f40095fae22af U drh -Z 7d7f3f0271fe8b8df2863a2c22053442 +Z be5219411fed69e2fe0e0626925bcd5e diff --git a/manifest.uuid b/manifest.uuid index 52d4aafd99..db15d6664e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a27861c28c4791e51d797aa37e9cca806cb58775 \ No newline at end of file +3e922208b68563489c7766abb9afb4885113e7b8 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d3a36ef484..d840a39a15 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1941,6 +1941,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ rc = SQLITE_OK; }else{ rc = sqlite3OsTruncate(pPager->jfd, 0); + if( rc==SQLITE_OK && pPager->fullSync ){ + /* Make sure the new file size is written into the inode right away. + ** Otherwise the journal might resurrect following a power loss and + ** cause the last transaction to roll back. See + ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773 + */ + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + } } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST From 9f07cf7b2e43548f60bd7497dafba30856df3531 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 15:27:05 +0000 Subject: [PATCH 17/17] Take steps to avoid misestimating range query costs based on STAT4 data due to the roundoff error of converting from integers to LogEst and back to integers. FossilOrigin-Name: 3c933bf95f291f7957580d823dce92c981375a5c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 1 + src/sqliteInt.h | 3 ++- src/where.c | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f4a8a2a556..8290b14522 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. -D 2014-10-21T21:56:06.890 +C Take\ssteps\sto\savoid\smisestimating\srange\squery\scosts\sbased\son\sSTAT4\sdata\ndue\sto\sthe\sroundoff\serror\sof\sconverting\sfrom\sintegers\sto\sLogEst\sand\sback\nto\sintegers. +D 2014-10-22T15:27:05.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 8c322e1ecc08909526dbd5ab4421889d05f2263d +F src/analyze.c 567c94b763b67f7abda06dbf0ba34b0343ed9447 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -232,7 +232,7 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h f7812f74f2d0c6041ef6b91a99c5a45f775dd408 +F src/sqliteInt.h d6d423b0f62846eb441236bc15417aeede2ebbdc F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 994b38c8697aad095878ef1e4860902df457427f +F src/where.c 5099c42e24c63969b3cf3b52e18c1a36cb841a34 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a27861c28c4791e51d797aa37e9cca806cb58775 -R 1375151e695773da263f40095fae22af +P 3e922208b68563489c7766abb9afb4885113e7b8 +R 5c508733aebac553de79291a7eabd709 U drh -Z be5219411fed69e2fe0e0626925bcd5e +Z a3d7104c7f173585d7e788d6453aa065 diff --git a/manifest.uuid b/manifest.uuid index db15d6664e..b84af28cee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e922208b68563489c7766abb9afb4885113e7b8 \ No newline at end of file +3c933bf95f291f7957580d823dce92c981375a5c \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 7d36f01318..67bba1a28a 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1599,6 +1599,7 @@ static void initAvgEq(Index *pIdx){ nRow = pIdx->aiRowEst[0]; nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; } + pIdx->nRowEst0 = nRow; /* Set nSum to the number of distinct (iCol+1) field prefixes that ** occur in the stat4 table for this index. Set sumEq to the sum of diff --git a/src/sqliteInt.h b/src/sqliteInt.h index cba89b03e7..5409f7086f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1801,7 +1801,8 @@ struct Index { int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ - tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this table */ + tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ + tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif }; diff --git a/src/where.c b/src/where.c index 7f51d00c87..f708d7fad7 100644 --- a/src/where.c +++ b/src/where.c @@ -2198,7 +2198,7 @@ static int whereRangeScanEst( /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; - iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); + iUpper = p->nRowEst0; }else{ /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */