From 837efb4cbe1fa71b991f42efa9f9057fcf4a7c6b Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 17 Feb 2024 01:12:58 +0000 Subject: [PATCH 01/15] The PRAGMA optimize command invokes ANALYZE if a table shrinks by 25 times in addition to if it grows by 25 times. FossilOrigin-Name: 40532ffba91cf332c1ea4add80184031a9d6e10514d2d9b9a6cfd613091b81f7 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/pragma.c | 13 ++++++++----- src/vdbe.c | 21 +++++++++++++++------ 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 8a1169a17b..3582a19a3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssubtype\son\sthe\svalue\scolumn\sfrom\sjson_each/json_tree\sfor\scases\nwhen\sthe\svalue\sis\san\sarray\sor\sobject.\s\sFix\sfor\sthe\sbug\sreported\sby\n[forum:/forumpost/ecb94cd210|forum\spost\secb94cd210]. -D 2024-02-16T21:30:08.580 +C The\sPRAGMA\soptimize\scommand\sinvokes\sANALYZE\sif\sa\stable\sshrinks\sby\s25\stimes\nin\saddition\sto\sif\sit\sgrows\sby\s25\stimes. +D 2024-02-17T01:12:58.951 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c 24584cc928538426e047db670f21f3816286812e48dfd391a40e788f14d3cea2 +F src/pragma.c 2a49dd92e9f14eb7433c06d02f5d02b7e2890b05117e7c5be103a9da1251ba0d F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 @@ -809,7 +809,7 @@ F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c 078f040366d5bd5f47658d045f901c768c1c636c6eaea121f3a1cbd63c3edb5b F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c dd340f5add924a8397401f7fe4faf2304ffd49ab5853d32ffc1736093b9c5080 +F src/vdbe.c a9b83d5ecd34d7e9d50dc7b643351541ddac74b5e2b19770cb3f2686cb815066 F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b @@ -2162,8 +2162,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 670174916c660b24ba70e96a42984eb65ee52da50e9828bdeca4c9ff4bf92e20 -R e9613550f9ab9d1defe8fcc96e4778a1 +P 1c33c5db2e05019d1a375109f79ad8588a3c17f81e4f4b8d66c880c3c860e87e +R 56c40cfd7fd1a612586d6162b2f4f953 +T *branch * optimize-after-shrink +T *sym-optimize-after-shrink * +T -sym-trunk * U drh -Z a380dd04602de7f2ac17d4bbe29cc262 +Z ebc951629700e2b49319b3c4b80fffe0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b30f95cb8a..942fd1732c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c33c5db2e05019d1a375109f79ad8588a3c17f81e4f4b8d66c880c3c860e87e \ No newline at end of file +40532ffba91cf332c1ea4add80184031a9d6e10514d2d9b9a6cfd613091b81f7 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 872177e131..367c9fba2f 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2453,17 +2453,20 @@ void sqlite3Pragma( if( (pTab->tabFlags & TF_MaybeReanalyze)==0 ) continue; /* Reanalyze if the table is 25 times larger than the last analysis */ - szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + szThreshold = pTab->nRowLogEst; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( !pIdx->hasStat1 ){ - szThreshold = 0; /* Always analyze if any index lacks statistics */ + szThreshold = -1; /* Always analyze if any index lacks statistics */ break; } } - if( szThreshold ){ + if( szThreshold>=0 ){ sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, - sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, + sqlite3VdbeCurrentAddr(v)+3+(opMask&1), + szThreshold>=46 ? szThreshold-46 : -1, + szThreshold+46); + sqlite3VdbeAddOp1(v, OP_Close, iTabCur); VdbeCoverage(v); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", diff --git a/src/vdbe.c b/src/vdbe.c index 1e0d776536..da3eeeae91 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -6192,28 +6192,37 @@ case OP_Last: { /* jump, ncycle */ break; } -/* Opcode: IfSmaller P1 P2 P3 * * +/* Opcode: IfSizeBetween P1 P2 P3 P4 * ** -** Estimate the number of rows in the table P1. Jump to P2 if that -** estimate is less than approximately 2**(0.1*P3). +** Let N be the approximate number of rows in the table or index +** with cursor P1 and let X be 10*log2(N) if N is positive or -1 +** if N is zero. Thus X will be within the range of -1 to 640, inclusive +** Jump to P2 if X is in between P3 and P4, inclusive. */ -case OP_IfSmaller: { /* jump */ +case OP_IfSizeBetween: { /* jump */ VdbeCursor *pC; BtCursor *pCrsr; int res; i64 sz; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p3>=-1 && pOp->p3<=640 ); + assert( pOp->p4.i>=-1 && pOp->p4.i<=640 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); if( rc ) goto abort_due_to_error; - if( res==0 ){ + if( res!=0 ){ + sz = -1; /* -Infinity encoding */ + }else{ sz = sqlite3BtreeRowCountEst(pCrsr); - if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1; + assert( sz>0 ); + sz = sqlite3LogEst((u64)sz); } + res = sz>=pOp->p3 && sz<=pOp->p4.i; VdbeBranchTaken(res!=0,2); if( res ) goto jump_to_p2; break; From 42eb6a91cdb5fe221171e7a0e588abd73d50542d Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 17 Feb 2024 16:39:52 +0000 Subject: [PATCH 02/15] Add new MASK bits to PRAGMA optimize: 0x70000. FossilOrigin-Name: dd4497062569eec9ddfdaa7d6394c83ff40a7a59c6697a161bc4ff6d8af3bb29 --- manifest | 19 +++++------ manifest.uuid | 2 +- src/build.c | 4 +-- src/pragma.c | 94 ++++++++++++++++++++++++++++++++++++--------------- src/vdbe.c | 21 +++++++++--- 5 files changed, 94 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 3582a19a3e..18e8630874 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sPRAGMA\soptimize\scommand\sinvokes\sANALYZE\sif\sa\stable\sshrinks\sby\s25\stimes\nin\saddition\sto\sif\sit\sgrows\sby\s25\stimes. -D 2024-02-17T01:12:58.951 +C Add\snew\sMASK\sbits\sto\sPRAGMA\soptimize:\s\s0x70000. +D 2024-02-17T16:39:52.117 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -681,7 +681,7 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c dfd351decf9f819393dae8fb7294cae69c3818069d27f47fee34d5b505e37b47 F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240 F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062 -F src/build.c 05c9eb387638982092988d2ef4c00a91b0c29baa1b85923f082ce0cf2903ea7e +F src/build.c 04f1bcee189f045ab086d84fee95db42cb49df82ff8e84af8136309ff3c8a75f F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 23331529e654be40ca97d171cbbffe9b3d4c71cc53b78fe5501230675952da8b @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c 2a49dd92e9f14eb7433c06d02f5d02b7e2890b05117e7c5be103a9da1251ba0d +F src/pragma.c c5b767e7538a808fac4f17f8891acb1f14b4eed3458d971ddea4dd82f7623e0c F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 @@ -809,7 +809,7 @@ F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c 078f040366d5bd5f47658d045f901c768c1c636c6eaea121f3a1cbd63c3edb5b F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c a9b83d5ecd34d7e9d50dc7b643351541ddac74b5e2b19770cb3f2686cb815066 +F src/vdbe.c 6d2aecc9ab2f9db557b4841775d016e2697ff6312242e9a492cdf3e2529dffa6 F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b @@ -2162,11 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1c33c5db2e05019d1a375109f79ad8588a3c17f81e4f4b8d66c880c3c860e87e -R 56c40cfd7fd1a612586d6162b2f4f953 -T *branch * optimize-after-shrink -T *sym-optimize-after-shrink * -T -sym-trunk * +P 40532ffba91cf332c1ea4add80184031a9d6e10514d2d9b9a6cfd613091b81f7 +R 518017150456f4f932816e90c763b561 U drh -Z ebc951629700e2b49319b3c4b80fffe0 +Z 4ff4c253b76c891ec5769cfa9bfe8766 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 942fd1732c..c4b9f73820 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -40532ffba91cf332c1ea4add80184031a9d6e10514d2d9b9a6cfd613091b81f7 \ No newline at end of file +dd4497062569eec9ddfdaa7d6394c83ff40a7a59c6697a161bc4ff6d8af3bb29 \ No newline at end of file diff --git a/src/build.c b/src/build.c index cfa2c886c5..15f8fe1d24 100644 --- a/src/build.c +++ b/src/build.c @@ -2923,11 +2923,11 @@ void sqlite3EndTable( /* Test for cycles in generated columns and illegal expressions ** in CHECK constraints and in DEFAULT clauses. */ if( p->tabFlags & TF_HasGenerated ){ - sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, + sqlite3VdbeAddOp4(v, OP_SqlExec, 0x0001, 0, 0, sqlite3MPrintf(db, "SELECT*FROM\"%w\".\"%w\"", db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } - sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, + sqlite3VdbeAddOp4(v, OP_SqlExec, 0x0001, 0, 0, sqlite3MPrintf(db, "PRAGMA \"%w\".integrity_check(%Q)", db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } diff --git a/src/pragma.c b/src/pragma.c index 367c9fba2f..40a31846bb 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2383,44 +2383,57 @@ void sqlite3Pragma( ** ** The optional argument is a bitmask of optimizations to perform: ** - ** 0x0001 Debugging mode. Do not actually perform any optimizations - ** but instead return one line of text for each optimization - ** that would have been done. Off by default. + ** 0x00001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. ** - ** 0x0002 Run ANALYZE on tables that might benefit. On by default. - ** See below for additional information. + ** 0x00002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. ** - ** 0x0004 (Not yet implemented) Record usage and performance - ** information from the current session in the - ** database file so that it will be available to "optimize" - ** pragmas run by future database connections. + ** 0x10000 Look at tables to see if they need to be reanalyzed + ** even if they have not been queried during the current + ** connection. Off by default. ** - ** 0x0008 (Not yet implemented) Create indexes that might have - ** been helpful to recent queries + ** 0x20000 Run all ANALYZE operations using an analysis_limit that + ** is the lessor of the current analysis_limit and the + ** SQLITE_DEFAULT_OPTIMIZE_LIMIT compile-time option, + ** nominally 400. Off by default. ** - ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all - ** of the optimizations listed above except Debug Mode, including new - ** optimizations that have not yet been invented. If new optimizations are - ** ever added that should be off by default, those off-by-default - ** optimizations will have bitmasks of 0x10000 or larger. + ** 0x40000 Tables become candidates for reanalysis if their size + ** grows or shrinks by 10x. Without this option, they + ** become candidates for reanalysis if their size grows + ** or shrinks by 25x. Off (25x mode) by default. + ** + ** The default MASK is and always shall be 0x0fffe. In the current + ** implementation, the default mask only covers the 0x00002 optimization, + ** though additional optimizations that are covered by 0x0fffe might be + ** added in the future. Optimizations that are off by default and must + ** be explicitly requested have masks of 0x10000 or greater. ** ** DETERMINATION OF WHEN TO RUN ANALYZE ** ** In the current implementation, a table is analyzed if only if all of ** the following are true: ** - ** (1) MASK bit 0x02 is set. + ** (1) MASK bit 0x00002 is set. ** - ** (2) The query planner used sqlite_stat1-style statistics for one or - ** more indexes of the table at some point during the lifetime of - ** the current connection. + ** (2) Either the 0x10000 MASK bit is set or else the query planner used + ** sqlite_stat1-style statistics for one or more indexes of the table + ** at some point during the lifetime of the current connection. ** ** (3) One or more indexes of the table are currently unanalyzed OR - ** the number of rows in the table has increased by 25 times or more + ** the number of rows in the table has increased or decreased + ** 25 times or more (10 times or more if the 0x40000 bit is set) ** since the last time ANALYZE was run. ** + ** (4) The table is an ordinary table, not a virtual table or view. + ** + ** (5) The table name does not begin with "sqlite_". + ** ** The rules for when tables are analyzed are likely to change in - ** future releases. + ** future releases. Future versions of SQLite might accept a string + ** literal argument to this pragma that contains a mnemonic description + ** of the options rather than a bitmap. */ case PragTyp_OPTIMIZE: { int iDbLast; /* Loop termination point for the schema loop */ @@ -2432,6 +2445,11 @@ void sqlite3Pragma( LogEst szThreshold; /* Size threshold above which reanalysis needed */ char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ u32 opMask; /* Mask of operations to perform */ + int nLimit; /* Analysis limit to use */ + +#ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT +# define SQLITE_DEFAULT_OPTIMIZE_LIMIT 400 +#endif if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -2439,6 +2457,14 @@ void sqlite3Pragma( }else{ opMask = 0xfffe; } + if( (opMask & 0x20000)==0 ){ + nLimit = 0; + }else if( db->nAnalysisLimit>0 + && db->nAnalysisLimitnTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ if( iDb==1 ) continue; @@ -2447,10 +2473,20 @@ void sqlite3Pragma( for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ pTab = (Table*)sqliteHashData(k); + /* This only works for ordinary tables */ + if( !IsOrdinaryTable(pTab) ) continue; + + /* Do not scan system tables */ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ) continue; + /* If table pTab has not been used in a way that would benefit from - ** having analysis statistics during the current session, then skip it. - ** This also has the effect of skipping virtual tables and views */ - if( (pTab->tabFlags & TF_MaybeReanalyze)==0 ) continue; + ** having analysis statistics during the current session, then skip it, + ** unless the 0x10000 MASK bit is set. */ + if( (pTab->tabFlags & TF_MaybeReanalyze)==0 + && (opMask & 0x10000)==0 + ){ + continue; + } /* Reanalyze if the table is 25 times larger than the last analysis */ szThreshold = pTab->nRowLogEst; @@ -2461,11 +2497,12 @@ void sqlite3Pragma( } } if( szThreshold>=0 ){ + LogEst iRange = (opMask & 0x40000) ? 33 : 46; sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, sqlite3VdbeCurrentAddr(v)+3+(opMask&1), - szThreshold>=46 ? szThreshold-46 : -1, - szThreshold+46); + szThreshold>=iRange ? szThreshold-iRange : -1, + szThreshold+iRange); sqlite3VdbeAddOp1(v, OP_Close, iTabCur); VdbeCoverage(v); } @@ -2476,7 +2513,8 @@ void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); }else{ - sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp4(v, OP_SqlExec, nLimit ? 0x02 : 00, nLimit, 0, + zSubSql, P4_DYNAMIC); } } } diff --git a/src/vdbe.c b/src/vdbe.c index da3eeeae91..3b52c13fa7 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -6922,11 +6922,18 @@ case OP_CreateBtree: { /* out2 */ break; } -/* Opcode: SqlExec * * * P4 * +/* Opcode: SqlExec P1 P2 * P4 * ** ** Run the SQL statement or statements specified in the P4 string. -** Disable Auth and Trace callbacks while those statements are running if -** P1 is true. +** +** The P1 parameter is a bitmask of options: +** +** 0x0001 Disable Auth and Trace callbacks while the statements +** in P4 are running. +** +** 0x0002 Set db->nAnalysisLimit to P2 while the statements in +** P4 are running. +** */ case OP_SqlExec: { char *zErr; @@ -6934,6 +6941,7 @@ case OP_SqlExec: { sqlite3_xauth xAuth; #endif u8 mTrace; + int savedAnalysisLimit; sqlite3VdbeIncrWriteCounter(p, 0); db->nSqlExec++; @@ -6942,18 +6950,23 @@ case OP_SqlExec: { xAuth = db->xAuth; #endif mTrace = db->mTrace; - if( pOp->p1 ){ + savedAnalysisLimit = db->nAnalysisLimit; + if( pOp->p1 & 0x0001 ){ #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = 0; #endif db->mTrace = 0; } + if( pOp->p1 & 0x0002 ){ + db->nAnalysisLimit = pOp->p2; + } rc = sqlite3_exec(db, pOp->p4.z, 0, 0, &zErr); db->nSqlExec--; #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = xAuth; #endif db->mTrace = mTrace; + db->nAnalysisLimit = savedAnalysisLimit; if( zErr || rc ){ sqlite3VdbeError(p, "%s", zErr); sqlite3_free(zErr); From d43e7adca7eb5d8c1ac6b345aa2470ed0f8b2c06 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 18 Feb 2024 01:12:22 +0000 Subject: [PATCH 03/15] Hold a transaction during PRAGMA optimize, for performance. FossilOrigin-Name: d13b79eae6df7f9d1f3b8062ddc75a12ff038196b3d752d2672a9925fa45ca56 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 7 +++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 18e8630874..e3c3328eb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sMASK\sbits\sto\sPRAGMA\soptimize:\s\s0x70000. -D 2024-02-17T16:39:52.117 +C Hold\sa\stransaction\sduring\sPRAGMA\soptimize,\sfor\sperformance. +D 2024-02-18T01:12:22.988 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c c5b767e7538a808fac4f17f8891acb1f14b4eed3458d971ddea4dd82f7623e0c +F src/pragma.c c7ef49d3332badd6e17ac46e0aca762d7973c1e28156bc480c47aa3e56ae4d03 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 40532ffba91cf332c1ea4add80184031a9d6e10514d2d9b9a6cfd613091b81f7 -R 518017150456f4f932816e90c763b561 +P dd4497062569eec9ddfdaa7d6394c83ff40a7a59c6697a161bc4ff6d8af3bb29 +R 5aca31e5331abdce830c908f0aade7ae U drh -Z 4ff4c253b76c891ec5769cfa9bfe8766 +Z 497aafa2ad4da05132856bc0e0f3518f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c4b9f73820..0aec312a78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dd4497062569eec9ddfdaa7d6394c83ff40a7a59c6697a161bc4ff6d8af3bb29 \ No newline at end of file +d13b79eae6df7f9d1f3b8062ddc75a12ff038196b3d752d2672a9925fa45ca56 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 40a31846bb..9bb971ed77 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2446,6 +2446,7 @@ void sqlite3Pragma( char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ u32 opMask; /* Mask of operations to perform */ int nLimit; /* Analysis limit to use */ + int once = 0; /* One-time initialization done */ #ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT # define SQLITE_DEFAULT_OPTIMIZE_LIMIT 400 @@ -2488,6 +2489,12 @@ void sqlite3Pragma( continue; } + /* Hold a write transaction open for efficiency */ + if( !once ){ + sqlite3BeginWriteOperation(pParse, 0, iDb); + once = 1; + } + /* Reanalyze if the table is 25 times larger than the last analysis */ szThreshold = pTab->nRowLogEst; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ From e0a9935be1c506646566f6b7845eb381bb219e16 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 19 Feb 2024 12:20:46 +0000 Subject: [PATCH 04/15] Extend the strerror_r() result type discrepancy check in os_unix.c to include the Android NDK, as reported in [forum post 3f13857fa4062301|forum:3f13857fa4062301]. The NDK's strerror_r() signature is version-dependent, whereas this change assumes int return (POSIX semantic) across all versions. FossilOrigin-Name: f18b2524da6bbbcf372b292df52fbe6efa49fd6e1f1f13ef2447279d559f7a08 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 6 +++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ab0f22bde5..1fd935940c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\srounding\sin\szero-precision\s%f\sand\s%g\sprintf\sconversions.\n[forum:/info/393708f4a8|Forum\spost\s393708f4a8].\s\sThis\sbug\swas\nintroduced\sby\scheck-in\s[32befb224b254639]\sand\sfirst\sappeared\sin\sversion\s3.43.0. -D 2024-02-17T03:32:31.878 +C Extend\sthe\sstrerror_r()\sresult\stype\sdiscrepancy\scheck\sin\sos_unix.c\sto\sinclude\sthe\sAndroid\sNDK,\sas\sreported\sin\s[forum\spost\s3f13857fa4062301|forum:3f13857fa4062301].\sThe\sNDK's\sstrerror_r()\ssignature\sis\sversion-dependent,\swhereas\sthis\schange\sassumes\sint\sreturn\s(POSIX\ssemantic)\sacross\sall\sversions. +D 2024-02-19T12:20:46.413 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -723,7 +723,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c fa9b81b642e60e77ffaf98bd1a2e5fde16c1c2317614ec178bf3bd5864772356 +F src/os_unix.c 8d7533b3b4d0d2d6ddd34d1ebc92f50a91f04e722a3a9295a000bc3c25128e2f F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1c33c5db2e05019d1a375109f79ad8588a3c17f81e4f4b8d66c880c3c860e87e -R e3e534a124d08ab0760f858683268942 -U drh -Z 25ef9b1be0189ee473aa53bd8732a56c +P 7fca1bc482fc2456d75392eb42f768fda72631c9070de46b8123b1126e78306f +R a2e48d1032c74dc590366e533d9ce522 +U stephan +Z f46d908a23e6be0d034c20949f0d6956 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1e28ec46dd..ed8ee25bd1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fca1bc482fc2456d75392eb42f768fda72631c9070de46b8123b1126e78306f \ No newline at end of file +f18b2524da6bbbcf372b292df52fbe6efa49fd6e1f1f13ef2447279d559f7a08 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 4b3d63c2c1..4663c22d94 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -1295,8 +1295,12 @@ static int unixLogErrorAtLine( ** available, the error message will often be an empty string. Not a ** huge problem. Incorrectly concluding that the GNU version is available ** could lead to a segfault though. + ** + ** Forum post 3f13857fa4062301 reports that the Android SDK may use + ** int-type return, depending on its version. */ -#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) +#if (defined(STRERROR_R_CHAR_P) || defined(__USE_GNU)) \ + && !defined(ANDROID) && !defined(__ANDROID__) zErr = # endif strerror_r(iErrno, aErr, sizeof(aErr)-1); From 9f34a05e98c3cffe48d935979bf745cc267efe4e Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 19 Feb 2024 13:06:27 +0000 Subject: [PATCH 05/15] Change the 0x20000 bit (use analysis limit) to 0x10, meaning that this feature is on by default. The default analysis limit is changed to 2000 which is almost always sufficient for accurate analysis results. FossilOrigin-Name: 4abd47b5917099a2f74e53e12c987da0722304a5e9a93b6d43015c1f45c48444 --- manifest | 14 +++++------ manifest.uuid | 2 +- src/pragma.c | 66 ++++++++++++++++++++++++++++++++++---------------- test/busy.test | 2 +- 4 files changed, 54 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index e3c3328eb1..54ba56837e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Hold\sa\stransaction\sduring\sPRAGMA\soptimize,\sfor\sperformance. -D 2024-02-18T01:12:22.988 +C Change\sthe\s0x20000\sbit\s(use\sanalysis\slimit)\sto\s0x10,\smeaning\sthat\sthis\sfeature\nis\son\sby\sdefault.\s\sThe\sdefault\sanalysis\slimit\sis\schanged\sto\s2000\swhich\sis\nalmost\salways\ssufficient\sfor\saccurate\sanalysis\sresults. +D 2024-02-19T13:06:27.586 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c c7ef49d3332badd6e17ac46e0aca762d7973c1e28156bc480c47aa3e56ae4d03 +F src/pragma.c b877efa88ea41560ee5dd95daa30c8cadb743381c59b8921a5143bf0792c6e3b F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 @@ -947,7 +947,7 @@ F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b F test/btree01.test fef17d9e999ac4f04095948e3438fbe674f4e07bb2c63bb1cad41d87baee077f F test/btree02.test 7555a5440453d900410160a52554fe6478af4faf53098f7235f1f443d5a1d6cc F test/btreefault.test a82a23b0578bc587afbf9a622c8f54a54f63762f062ba8a35613cfee38ab42f9 -F test/busy.test 510dc6daaad18bcbbc085bcc6217d6dc418def5e73f72ce1475eea0cb7834727 +F test/busy.test caff7164c16ce06a53af51f9e4c2753d4cc64250e00790a5e48b9c4f4be37597 F test/busy2.test 20823a5d7c42fb257d9f108c66312d90b1bb4ec3d80ba6b4e371073727560f98 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61 @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dd4497062569eec9ddfdaa7d6394c83ff40a7a59c6697a161bc4ff6d8af3bb29 -R 5aca31e5331abdce830c908f0aade7ae +P d13b79eae6df7f9d1f3b8062ddc75a12ff038196b3d752d2672a9925fa45ca56 +R e258fa6228ef566d2b5670b2822a20d1 U drh -Z 497aafa2ad4da05132856bc0e0f3518f +Z e864b92d6fb5e0c6f09ca55fa83dd960 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0aec312a78..9cda73c43e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d13b79eae6df7f9d1f3b8062ddc75a12ff038196b3d752d2672a9925fa45ca56 \ No newline at end of file +4abd47b5917099a2f74e53e12c987da0722304a5e9a93b6d43015c1f45c48444 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 9bb971ed77..6e1fc8bc00 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -30,6 +30,34 @@ ** ../tool/mkpragmatab.tcl. */ #include "pragma.h" +/* +** When the 0x10 bit of PRAGMA optimize is set, any ANALYZE commands +** will be run with an analysis_limit set to the lessor of the value of +** the following macro or to the actual analysis_limit if it is non-zero, +** in order to prevent PRAGMA optimize from running for too long. +** +** The value of 2000 is chosen emperically so that the worst-case run-time +** for PRAGMA optimize does not exceed 100 milliseconds against a variety +** of test databases on a RaspberryPI-4 compiled using -Os and without +** -DSQLITE_DEBUG. Of course, your mileage may vary. For the purpose of +** his paragraph, "worst-case" means that ANALYZE ends up being +** run on every table in the database. The worst case typically only +** happens if PRAGMA optimize is run on a database file for which ANALYZE +** has not been previously run and the 0x10000 flag is included so that +** all tables are analyzed. The usual case for PRAGMA optimize is that +** no ANALYZE commands will be run at all, or if any ANALYZE happens it +** will be against a single table, so that expected timing for PRAGMA +** optimize on a PI-4 is more like 1 millisecond or less with the 0x10000 +** flag or less than 100 microseconds without the 0x10000 flag. +** +** An analysis limit of 2000 is almost always sufficient for the query +** planner to fully characterize an index. The additional accuracy from +** a larger analysis is not usually helpful. +*/ +#ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT +# define SQLITE_DEFAULT_OPTIMIZE_LIMIT 2000 +#endif + /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or @@ -2390,20 +2418,19 @@ void sqlite3Pragma( ** 0x00002 Run ANALYZE on tables that might benefit. On by default. ** See below for additional information. ** + ** 0x00010 Run all ANALYZE operations using an analysis_limit that + ** is the lessor of the current analysis_limit and the + ** SQLITE_DEFAULT_OPTIMIZE_LIMIT compile-time option. + ** The default value of SQLITE_DEFAULT_OPTIMIZE_LIMIT is + ** currently (2024-02-19) set to 2000, which is such that + ** the worst case run-time for PRAGMA optimize on a 100MB + ** database will usually be less than 100 milliseconds on + ** a RaspberryPI-4 class machine. Off by default. + ** ** 0x10000 Look at tables to see if they need to be reanalyzed ** even if they have not been queried during the current ** connection. Off by default. ** - ** 0x20000 Run all ANALYZE operations using an analysis_limit that - ** is the lessor of the current analysis_limit and the - ** SQLITE_DEFAULT_OPTIMIZE_LIMIT compile-time option, - ** nominally 400. Off by default. - ** - ** 0x40000 Tables become candidates for reanalysis if their size - ** grows or shrinks by 10x. Without this option, they - ** become candidates for reanalysis if their size grows - ** or shrinks by 25x. Off (25x mode) by default. - ** ** The default MASK is and always shall be 0x0fffe. In the current ** implementation, the default mask only covers the 0x00002 optimization, ** though additional optimizations that are covered by 0x0fffe might be @@ -2422,9 +2449,9 @@ void sqlite3Pragma( ** at some point during the lifetime of the current connection. ** ** (3) One or more indexes of the table are currently unanalyzed OR - ** the number of rows in the table has increased or decreased - ** 25 times or more (10 times or more if the 0x40000 bit is set) - ** since the last time ANALYZE was run. + ** the number of rows in the table has increased or decreased by + ** 10-fold (the new size is either greater than 10 times the old + ** size or less than 1/10th of the old size). ** ** (4) The table is an ordinary table, not a virtual table or view. ** @@ -2448,17 +2475,13 @@ void sqlite3Pragma( int nLimit; /* Analysis limit to use */ int once = 0; /* One-time initialization done */ -#ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT -# define SQLITE_DEFAULT_OPTIMIZE_LIMIT 400 -#endif - if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); if( (opMask & 0x02)==0 ) break; }else{ opMask = 0xfffe; } - if( (opMask & 0x20000)==0 ){ + if( (opMask & 0x10)==0 ){ nLimit = 0; }else if( db->nAnalysisLimit>0 && db->nAnalysisLimitnRowLogEst; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( !pIdx->hasStat1 ){ @@ -2504,7 +2528,7 @@ void sqlite3Pragma( } } if( szThreshold>=0 ){ - LogEst iRange = (opMask & 0x40000) ? 33 : 46; + LogEst iRange = 33; /* 10x size change */ sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, sqlite3VdbeCurrentAddr(v)+3+(opMask&1), diff --git a/test/busy.test b/test/busy.test index be0515b013..896c7fa651 100644 --- a/test/busy.test +++ b/test/busy.test @@ -106,7 +106,7 @@ do_test 3.4 { proc busy_handler {n} { return 1 } do_test 3.5 { catchsql { PRAGMA optimize } -} {0 {}} +} {1 {database is locked}} do_test 3.6 { execsql { COMMIT } db2 From 6c6356f7f2f3c6e717d445d117407e2edecccac5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 19 Feb 2024 13:50:09 +0000 Subject: [PATCH 06/15] Simplifications to PRAGMA optimize to make it easier to use. It always tries to ANALYZE unanalyzed indexes. The 0x10000 flag just makes it check for size changes in all tables. FossilOrigin-Name: 44ed7f4cd07a88a2fdd303a2c78e6babe01d7344b399bd2b80ed68d75a77aaa2 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/pragma.c | 85 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 61 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 54ba56837e..fdb16737f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\s0x20000\sbit\s(use\sanalysis\slimit)\sto\s0x10,\smeaning\sthat\sthis\sfeature\nis\son\sby\sdefault.\s\sThe\sdefault\sanalysis\slimit\sis\schanged\sto\s2000\swhich\sis\nalmost\salways\ssufficient\sfor\saccurate\sanalysis\sresults. -D 2024-02-19T13:06:27.586 +C Simplifications\sto\sPRAGMA\soptimize\sto\smake\sit\seasier\sto\suse.\s\sIt\salways\ntries\sto\sANALYZE\sunanalyzed\sindexes.\s\sThe\s0x10000\sflag\sjust\smakes\sit\scheck\nfor\ssize\schanges\sin\sall\stables. +D 2024-02-19T13:50:09.987 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c b877efa88ea41560ee5dd95daa30c8cadb743381c59b8921a5143bf0792c6e3b +F src/pragma.c 9197331dd283a6010bb4e2c129a9015f854203d27fa6fbe89e1aea742b78351c F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d13b79eae6df7f9d1f3b8062ddc75a12ff038196b3d752d2672a9925fa45ca56 -R e258fa6228ef566d2b5670b2822a20d1 +P 4abd47b5917099a2f74e53e12c987da0722304a5e9a93b6d43015c1f45c48444 +R 39db09ba2f5384fee677814cc2cbe4fe U drh -Z e864b92d6fb5e0c6f09ca55fa83dd960 +Z cf23276671f3baaf4e4f3579488af586 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9cda73c43e..f1cc73e3f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4abd47b5917099a2f74e53e12c987da0722304a5e9a93b6d43015c1f45c48444 \ No newline at end of file +44ed7f4cd07a88a2fdd303a2c78e6babe01d7344b399bd2b80ed68d75a77aaa2 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 6e1fc8bc00..53dc9ced30 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2425,11 +2425,15 @@ void sqlite3Pragma( ** currently (2024-02-19) set to 2000, which is such that ** the worst case run-time for PRAGMA optimize on a 100MB ** database will usually be less than 100 milliseconds on - ** a RaspberryPI-4 class machine. Off by default. + ** a RaspberryPI-4 class machine. On by default. + ** + ** 0x00020 Run ANALYZE on any table that has a complete index + ** (an index without a WHERE clause) that lacks an entry + ** in the sqlite_stat1 table. On by default. ** ** 0x10000 Look at tables to see if they need to be reanalyzed - ** even if they have not been queried during the current - ** connection. Off by default. + ** due to growth or shrinkage even if they have not been + ** queried during the current connection. Off by default. ** ** The default MASK is and always shall be 0x0fffe. In the current ** implementation, the default mask only covers the 0x00002 optimization, @@ -2444,18 +2448,25 @@ void sqlite3Pragma( ** ** (1) MASK bit 0x00002 is set. ** - ** (2) Either the 0x10000 MASK bit is set or else the query planner used - ** sqlite_stat1-style statistics for one or more indexes of the table - ** at some point during the lifetime of the current connection. + ** (2) The table is an ordinary table, not a virtual table or view. ** - ** (3) One or more indexes of the table are currently unanalyzed OR - ** the number of rows in the table has increased or decreased by - ** 10-fold (the new size is either greater than 10 times the old - ** size or less than 1/10th of the old size). + ** (3) The table name does not begin with "sqlite_". ** - ** (4) The table is an ordinary table, not a virtual table or view. + ** (4) One or more of the following is true: + ** (4a) The 0x10000 MASK bit is set. + ** (4b) One or more complete indexes on the table lacks an entry + ** in the sqlite_stat1 table. + ** (4c) The query planner used sqlite_stat1-style statistics for one + ** or more indexes of the tableat some point during the lifetime + ** of the current connection. ** - ** (5) The table name does not begin with "sqlite_". + ** (5) One or more of the following is true: + ** (5a) One or mroe complete indexes on the table lacks an entry + ** in the sqlite_stat1 table. (Same as 4a) + ** (5b) The number of rows in the table has increased or decreased by + ** 10-fold. In other words, the current size of the table is + ** 10 times larger than the size in sqlite_stat1 or else the + ** current size is less than 1/10th the size in sqlite_stat1. ** ** The rules for when tables are analyzed are likely to change in ** future releases. Future versions of SQLite might accept a string @@ -2473,7 +2484,7 @@ void sqlite3Pragma( char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ u32 opMask; /* Mask of operations to perform */ int nLimit; /* Analysis limit to use */ - int once = 0; /* One-time initialization done */ + int nCheck = 0; /* Number of tables to be optimized */ if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -2503,30 +2514,42 @@ void sqlite3Pragma( /* Do not scan system tables */ if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ) continue; - /* If table pTab has not been used in a way that would benefit from - ** having analysis statistics during the current session, then skip it, - ** unless the 0x10000 MASK bit is set. */ - if( (pTab->tabFlags & TF_MaybeReanalyze)==0 - && (opMask & 0x10000)==0 - ){ - continue; - } - - /* Hold a write transaction open for efficiency */ - if( !once || 1 ){ - sqlite3BeginWriteOperation(pParse, 0, iDb); - once = 1; - } - - /* Reanalyze if the table is 10 times larger or smaller than - ** the last analysis */ + /* Find the size of the table as last recorded in sqlite_stat1. + ** If any complete index (index without a WHERE clause) is unanalyzed, + ** then the threshold is -1 to indicate a new, unanalyzed index + */ szThreshold = pTab->nRowLogEst; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !pIdx->hasStat1 ){ + if( !pIdx->hasStat1 && pIdx->pPartIdxWhere==0 ){ szThreshold = -1; /* Always analyze if any index lacks statistics */ break; } } + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it, + ** unless the 0x10000 MASK bit is set. */ + if( (pTab->tabFlags & TF_MaybeReanalyze)!=0 ){ + /* Check for size change if stat1 has been used for a query */ + }else if( opMask & 0x10000 ){ + /* Check for size change if 0x10000 is set */ + }else if( pTab->pIndex!=0 && szThreshold<0 ){ + /* Do analysis if unanalyzed complete indexes exists */ + }else{ + /* Otherwise, we can skip this table */ + continue; + } + + nCheck++; + if( nCheck==2 ){ + /* If ANALYZE might be invoked two or more times, hold a write + ** transaction for efficiency */ + sqlite3BeginWriteOperation(pParse, 0, iDb); + } + + /* Reanalyze if the table is 10 times larger or smaller than + ** the last analysis. Unconditional reanalysis if there are + ** unanalyzed complete indexes. */ if( szThreshold>=0 ){ LogEst iRange = 33; /* 10x size change */ sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); From e7bdb2172c717f7ba64e15fb65b930524e759d71 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 19 Feb 2024 16:22:58 +0000 Subject: [PATCH 07/15] If a table has one or more rows and it has a partial index has zero rows, still make an entry in the sqlite_stat1 table for the partial index, so that we know that "PRAGMA optimize" does not need to redo the whole table. FossilOrigin-Name: e147b18991dd462fff367442acb0504fdf193a31843ed34ec8c1ced30747bf8a --- ext/session/sessionstat1.test | 4 +- manifest | 20 +++++----- manifest.uuid | 2 +- src/analyze.c | 74 ++++++++++++++++++++--------------- src/btree.c | 8 ++-- src/pragma.c | 19 +++++---- src/vdbe.c | 2 +- 7 files changed, 70 insertions(+), 59 deletions(-) diff --git a/ext/session/sessionstat1.test b/ext/session/sessionstat1.test index 2757d60440..16dc4e2727 100644 --- a/ext/session/sessionstat1.test +++ b/ext/session/sessionstat1.test @@ -92,7 +92,7 @@ do_test 2.1 { } {} do_execsql_test -db db2 2.2 { - SELECT * FROM sqlite_stat1 + SELECT * FROM sqlite_stat1 ORDER BY tbl, idx } { t1 sqlite_autoindex_t1_1 {32 1} t1 t1b {32 4} @@ -104,7 +104,7 @@ do_test 2.3 { } {} do_execsql_test -db db2 2.4 { - SELECT * FROM sqlite_stat1 + SELECT * FROM sqlite_stat1 ORDER BY tbl, idx; } { t1 sqlite_autoindex_t1_1 {32 1} t1 t1b {32 4} diff --git a/manifest b/manifest index fdb16737f1..c12c473dca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplifications\sto\sPRAGMA\soptimize\sto\smake\sit\seasier\sto\suse.\s\sIt\salways\ntries\sto\sANALYZE\sunanalyzed\sindexes.\s\sThe\s0x10000\sflag\sjust\smakes\sit\scheck\nfor\ssize\schanges\sin\sall\stables. -D 2024-02-19T13:50:09.987 +C If\sa\stable\shas\sone\sor\smore\srows\sand\sit\shas\sa\spartial\sindex\shas\szero\srows,\nstill\smake\san\sentry\sin\sthe\ssqlite_stat1\stable\sfor\sthe\spartial\sindex,\sso\sthat\nwe\sknow\sthat\s"PRAGMA\soptimize"\sdoes\snot\sneed\sto\sredo\sthe\swhole\stable. +D 2024-02-19T16:22:58.040 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -567,7 +567,7 @@ F ext/session/sessionnoop2.test de4672dce88464396ec9f30ed08c6c01643a69c53ae540fa F ext/session/sessionrebase.test 702378bdcb5062f1106e74457beca8797d09c113a81768734a58b197b5b334e2 F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a80600a44396f7363 F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 -F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c +F ext/session/sessionstat1.test 5e718d5888c0c49bbb33a7a4f816366db85f59f6a4f97544a806421b85dc2dec F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc F ext/session/sqlite3session.c 829d468f0f3d2710aace56b0116a7ca3f414683ce78e3125ae5e21547a895078 F ext/session/sqlite3session.h 4cf19a51975746d7cff2fdd74db8b769c570958e1c3639ac150d824ac1553b3e @@ -672,13 +672,13 @@ F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 30c2333b8bb3af71e4eb9adeadee8aa20edb15917ed44b8422e5cd15f3dfcddc -F src/analyze.c 0f15753308c3bca7674f31fa7e0807ffcb8b120c36eef7d00b62b33079ddc854 +F src/analyze.c dacc8f062bbda02c8ed3bbca0ab6122de010d094101d568df266a7b0d665d74f F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 -F src/btree.c dfd351decf9f819393dae8fb7294cae69c3818069d27f47fee34d5b505e37b47 +F src/btree.c 2f9dcf7c27879edb7897354029343bcab0c5a3dd2bde3bf3fd8f814b71344d22 F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240 F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062 F src/build.c 04f1bcee189f045ab086d84fee95db42cb49df82ff8e84af8136309ff3c8a75f @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c 9197331dd283a6010bb4e2c129a9015f854203d27fa6fbe89e1aea742b78351c +F src/pragma.c 0bd3737acf69282fb1733e170a88a5caf3dba088092665e603ee5a6d651c0e2f F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 @@ -809,7 +809,7 @@ F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c 078f040366d5bd5f47658d045f901c768c1c636c6eaea121f3a1cbd63c3edb5b F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c 6d2aecc9ab2f9db557b4841775d016e2697ff6312242e9a492cdf3e2529dffa6 +F src/vdbe.c 805af8bec9cae76a873b0ec4c91c3d1bd49b38e86c1c533ed87ec7a07a5042a5 F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4abd47b5917099a2f74e53e12c987da0722304a5e9a93b6d43015c1f45c48444 -R 39db09ba2f5384fee677814cc2cbe4fe +P 44ed7f4cd07a88a2fdd303a2c78e6babe01d7344b399bd2b80ed68d75a77aaa2 +R 425f4513952a4b2a7a3cf3f40b480a46 U drh -Z cf23276671f3baaf4e4f3579488af586 +Z 08209902059a589603329b04ece67a70 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f1cc73e3f0..5ae06f96c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -44ed7f4cd07a88a2fdd303a2c78e6babe01d7344b399bd2b80ed68d75a77aaa2 \ No newline at end of file +e147b18991dd462fff367442acb0504fdf193a31843ed34ec8c1ced30747bf8a \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 59e3d98377..e7c1068ae0 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -872,7 +872,7 @@ static void statGet( if( iVal==2 && p->nRow*10 <= nDistinct*11 ) iVal = 1; sqlite3_str_appendf(&sStat, " %llu", iVal); #ifdef SQLITE_ENABLE_STAT4 - assert( p->current.anEq[i] ); + assert( p->current.anEq[i] || p->nRow==0 ); #endif } sqlite3ResultStrAccum(context, &sStat); @@ -1057,7 +1057,7 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ - int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrGotoEnd; /* Address of "OP_Rewind iIdxCur" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ int nColTest; /* Number of columns to test for changes */ @@ -1081,9 +1081,14 @@ static void analyzeOneTable( /* ** Pseudo-code for loop that calls stat_push(): ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() ** goto chng_addr_0; ** ** next_row: @@ -1122,41 +1127,36 @@ static void analyzeOneTable( sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); - /* Invoke the stat_init() function. The arguments are: - ** + /* Implementation of the following: + ** + ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() + ** goto chng_addr_0; + */ + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + + /* Arguments to stat_init(): ** (1) the number of columns in the index including the rowid ** (or for a WITHOUT ROWID table, the number of PK columns), ** (2) the number of columns in the key without the rowid/pk - ** (3) estimated number of rows in the index, - */ + ** (3) estimated number of rows in the index. */ sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); assert( regRowid==regStat+2 ); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); -#ifdef SQLITE_ENABLE_STAT4 - if( OptimizationEnabled(db, SQLITE_Stat4) ){ - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - }else -#endif - { - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); - } - assert( regTemp2==regStat+4 ); - sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, + OptimizationDisabled(db, SQLITE_Stat4)); sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, &statInitFuncdef, 0); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); - /* Implementation of the following: - ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; - ** regChng = 0 - ** goto next_push_0; - ** - */ sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); addrNextRow = sqlite3VdbeCurrentAddr(v); @@ -1263,6 +1263,12 @@ static void analyzeOneTable( } /* Add the entry to the stat1 table. */ + if( pIdx->pPartIdxWhere ){ + /* Partial indexes might get a zero-entry in sqlite_stat1. But + ** an empty table is omitted from sqlite_stat1. */ + sqlite3VdbeJumpHere(v, addrGotoEnd); + addrGotoEnd = 0; + } callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); assert( "BBB"[0]==SQLITE_AFF_TEXT ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); @@ -1286,6 +1292,12 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + /* No STAT4 data is generated if the number of rows is zero */ + if( addrGotoEnd==0 ){ + sqlite3VdbeAddOp2(v, OP_Cast, regStat1, SQLITE_AFF_INTEGER); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); + } + if( doOnce ){ int mxCol = nCol; Index *pX; @@ -1338,7 +1350,7 @@ static void analyzeOneTable( #endif /* SQLITE_ENABLE_STAT4 */ /* End of analysis */ - sqlite3VdbeJumpHere(v, addrRewind); + if( addrGotoEnd ) sqlite3VdbeJumpHere(v, addrGotoEnd); } diff --git a/src/btree.c b/src/btree.c index 49c07c81a6..16b683abe7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6182,10 +6182,10 @@ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - /* Currently this interface is only called by the OP_IfSmaller - ** opcode, and it that case the cursor will always be valid and - ** will always point to a leaf node. */ - if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + /* Currently this interface is only called by the OP_IfSizeBetween + ** opcode and the OP_Count opcode with P3=1. In either case, + ** the cursor will always be valid unless the btree is empty. */ + if( pCur->eState!=CURSOR_VALID ) return 0; if( NEVER(pCur->pPage->leaf==0) ) return -1; n = pCur->pPage->nCell; diff --git a/src/pragma.c b/src/pragma.c index 53dc9ced30..77ec9a57e2 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2427,9 +2427,8 @@ void sqlite3Pragma( ** database will usually be less than 100 milliseconds on ** a RaspberryPI-4 class machine. On by default. ** - ** 0x00020 Run ANALYZE on any table that has a complete index - ** (an index without a WHERE clause) that lacks an entry - ** in the sqlite_stat1 table. On by default. + ** 0x00020 Run ANALYZE on any table that has a index that lacks an + ** entry in the sqlite_stat1 table. On by default. ** ** 0x10000 Look at tables to see if they need to be reanalyzed ** due to growth or shrinkage even if they have not been @@ -2454,14 +2453,14 @@ void sqlite3Pragma( ** ** (4) One or more of the following is true: ** (4a) The 0x10000 MASK bit is set. - ** (4b) One or more complete indexes on the table lacks an entry + ** (4b) One or more indexes on the table lacks an entry ** in the sqlite_stat1 table. ** (4c) The query planner used sqlite_stat1-style statistics for one ** or more indexes of the tableat some point during the lifetime ** of the current connection. ** ** (5) One or more of the following is true: - ** (5a) One or mroe complete indexes on the table lacks an entry + ** (5a) One or mroe indexes on the table lacks an entry ** in the sqlite_stat1 table. (Same as 4a) ** (5b) The number of rows in the table has increased or decreased by ** 10-fold. In other words, the current size of the table is @@ -2515,12 +2514,12 @@ void sqlite3Pragma( if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ) continue; /* Find the size of the table as last recorded in sqlite_stat1. - ** If any complete index (index without a WHERE clause) is unanalyzed, - ** then the threshold is -1 to indicate a new, unanalyzed index + ** If any index is unanalyzed, then the threshold is -1 to + ** indicate a new, unanalyzed index */ szThreshold = pTab->nRowLogEst; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !pIdx->hasStat1 && pIdx->pPartIdxWhere==0 ){ + if( !pIdx->hasStat1 ){ szThreshold = -1; /* Always analyze if any index lacks statistics */ break; } @@ -2534,7 +2533,7 @@ void sqlite3Pragma( }else if( opMask & 0x10000 ){ /* Check for size change if 0x10000 is set */ }else if( pTab->pIndex!=0 && szThreshold<0 ){ - /* Do analysis if unanalyzed complete indexes exists */ + /* Do analysis if unanalyzed indexes exists */ }else{ /* Otherwise, we can skip this table */ continue; @@ -2549,7 +2548,7 @@ void sqlite3Pragma( /* Reanalyze if the table is 10 times larger or smaller than ** the last analysis. Unconditional reanalysis if there are - ** unanalyzed complete indexes. */ + ** unanalyzed indexes. */ if( szThreshold>=0 ){ LogEst iRange = 33; /* 10x size change */ sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); diff --git a/src/vdbe.c b/src/vdbe.c index 3b52c13fa7..48a029dbf8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2086,7 +2086,7 @@ case OP_RealAffinity: { /* in1 */ } #endif -#ifndef SQLITE_OMIT_CAST +#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_ANALYZE) /* Opcode: Cast P1 P2 * * * ** Synopsis: affinity(r[P1]) ** From 74b0aad09f0931c6645a1c43a0290ab1ee6ab4ff Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 19 Feb 2024 19:56:40 +0000 Subject: [PATCH 08/15] In PRAGMA optimize, scale back the analysis_limit when many different tables and indexes must be scanned. FossilOrigin-Name: 636615358aebfef80a657d09f15f608df9e64882b2aaa5574ce31a4f1c71fd70 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 22 +++++++++++++++++++++- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 690d3adac6..aef3dc3eee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sinto\sthe\sbetter-optimize\sbranch. -D 2024-02-19T18:55:32.241 +C In\sPRAGMA\soptimize,\sscale\sback\sthe\sanalysis_limit\swhen\smany\sdifferent\stables\nand\sindexes\smust\sbe\sscanned. +D 2024-02-19T19:56:40.420 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c 0bd3737acf69282fb1733e170a88a5caf3dba088092665e603ee5a6d651c0e2f +F src/pragma.c c72ba9c8b300d713a318e7d199bc1d3f717f961c75de403f8db291841f2f58aa F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c d3392b2a20ee314ddeef34fb43c904bf4619eb20ff9a9e07e3950a7e4dcd6912 @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e147b18991dd462fff367442acb0504fdf193a31843ed34ec8c1ced30747bf8a f18b2524da6bbbcf372b292df52fbe6efa49fd6e1f1f13ef2447279d559f7a08 -R fada2b9bf9b286f261a00cb9eda0fee2 +P 2cf78a5b5fce7f2d49ce185d27f50722c91a1d6c91bbcecebd4bb5244a76769f +R c2c83408fc346dde3baee32faaf6bad7 U drh -Z 513df274231a3535cbd843c1569f620b +Z ec6824937a1b6fb54f3b9fdcf77219f7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b04ec72a34..aa9236de91 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2cf78a5b5fce7f2d49ce185d27f50722c91a1d6c91bbcecebd4bb5244a76769f \ No newline at end of file +636615358aebfef80a657d09f15f608df9e64882b2aaa5574ce31a4f1c71fd70 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 77ec9a57e2..18fc88aceb 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2460,7 +2460,7 @@ void sqlite3Pragma( ** of the current connection. ** ** (5) One or more of the following is true: - ** (5a) One or mroe indexes on the table lacks an entry + ** (5a) One or more indexes on the table lacks an entry ** in the sqlite_stat1 table. (Same as 4a) ** (5b) The number of rows in the table has increased or decreased by ** 10-fold. In other words, the current size of the table is @@ -2484,6 +2484,8 @@ void sqlite3Pragma( u32 opMask; /* Mask of operations to perform */ int nLimit; /* Analysis limit to use */ int nCheck = 0; /* Number of tables to be optimized */ + int nBtree = 0; /* Number of btrees to scan */ + int nIndex; /* Number of indexes on the current table */ if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -2518,7 +2520,9 @@ void sqlite3Pragma( ** indicate a new, unanalyzed index */ szThreshold = pTab->nRowLogEst; + nIndex = 0; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + nIndex++; if( !pIdx->hasStat1 ){ szThreshold = -1; /* Always analyze if any index lacks statistics */ break; @@ -2545,6 +2549,7 @@ void sqlite3Pragma( ** transaction for efficiency */ sqlite3BeginWriteOperation(pParse, 0, iDb); } + nBtree += nIndex+1; /* Reanalyze if the table is 10 times larger or smaller than ** the last analysis. Unconditional reanalysis if there are @@ -2572,6 +2577,21 @@ void sqlite3Pragma( } } sqlite3VdbeAddOp0(v, OP_Expire); + + /* In a schema with a large number of tables and indexes, scale back + ** the analysis_limit to avoid excess run-time in the worst case. + */ + if( !db->mallocFailed && nBtree>100 ){ + int iAddr, iEnd; + VdbeOp *aOp; + int nLimit = 100*SQLITE_DEFAULT_OPTIMIZE_LIMIT/nBtree; + if( nLimit<100 ) nLimit = 100; + aOp = sqlite3VdbeGetOp(v, 0); + iEnd = sqlite3VdbeCurrentAddr(v); + for(iAddr=0; iAddr Date: Mon, 19 Feb 2024 20:12:30 +0000 Subject: [PATCH 09/15] If there is no mention of a table in sqlite_stat1, use OP_Rewind to see if the table is empty prior to invoking ANALYZE. FossilOrigin-Name: 0cc93b19de597866292e0696f89fbd0e22a6cbb374678f5cc096fb889210fe3c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 15 +++++++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index aef3dc3eee..0449436b9d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sPRAGMA\soptimize,\sscale\sback\sthe\sanalysis_limit\swhen\smany\sdifferent\stables\nand\sindexes\smust\sbe\sscanned. -D 2024-02-19T19:56:40.420 +C If\sthere\sis\sno\smention\sof\sa\stable\sin\ssqlite_stat1,\suse\sOP_Rewind\sto\ssee\sif\nthe\stable\sis\sempty\sprior\sto\sinvoking\sANALYZE. +D 2024-02-19T20:12:30.657 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c c72ba9c8b300d713a318e7d199bc1d3f717f961c75de403f8db291841f2f58aa +F src/pragma.c 280945eb4a7b78c82cd1b4d0bdefbb69af685a654f8fbfeb3bfd7dc8fede5fda F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c d3392b2a20ee314ddeef34fb43c904bf4619eb20ff9a9e07e3950a7e4dcd6912 @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2cf78a5b5fce7f2d49ce185d27f50722c91a1d6c91bbcecebd4bb5244a76769f -R c2c83408fc346dde3baee32faaf6bad7 +P 636615358aebfef80a657d09f15f608df9e64882b2aaa5574ce31a4f1c71fd70 +R 98db8daafbcb183bee3a0d5c6d6f3dec U drh -Z ec6824937a1b6fb54f3b9fdcf77219f7 +Z 53d562a05375d893c6e8245ccbe9b40a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aa9236de91..b4d235c325 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -636615358aebfef80a657d09f15f608df9e64882b2aaa5574ce31a4f1c71fd70 \ No newline at end of file +0cc93b19de597866292e0696f89fbd0e22a6cbb374678f5cc096fb889210fe3c \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 18fc88aceb..e65233389e 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2486,6 +2486,7 @@ void sqlite3Pragma( int nCheck = 0; /* Number of tables to be optimized */ int nBtree = 0; /* Number of btrees to scan */ int nIndex; /* Number of indexes on the current table */ + int hasStat1; /* True if any STAT1 info available for the table */ if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -2520,10 +2521,13 @@ void sqlite3Pragma( ** indicate a new, unanalyzed index */ szThreshold = pTab->nRowLogEst; + hasStat1 = (pTab->tabFlags & TF_HasStat1)!=0; nIndex = 0; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ nIndex++; - if( !pIdx->hasStat1 ){ + if( pIdx->hasStat1 ){ + hasStat1 = 1; + }else{ szThreshold = -1; /* Always analyze if any index lacks statistics */ break; } @@ -2555,14 +2559,17 @@ void sqlite3Pragma( ** the last analysis. Unconditional reanalysis if there are ** unanalyzed indexes. */ if( szThreshold>=0 ){ - LogEst iRange = 33; /* 10x size change */ + const LogEst iRange = 33; /* 10x size change */ sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, - sqlite3VdbeCurrentAddr(v)+3+(opMask&1), + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold>=iRange ? szThreshold-iRange : -1, szThreshold+iRange); - sqlite3VdbeAddOp1(v, OP_Close, iTabCur); VdbeCoverage(v); + }else if( !hasStat1 ){ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp2(v, OP_Rewind, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1)); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", db->aDb[iDb].zDbSName, pTab->zName); From a1507a3fa02b8789665e326c8c014f1e58fcd80d Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 19 Feb 2024 20:47:59 +0000 Subject: [PATCH 10/15] Fix a minor hiccup in the computation of the number of btrees to be scanned. FossilOrigin-Name: 91302d9b2720d6b5184e1c24db55d351544adfde4b3d2ddcdc1561f79f11985a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0449436b9d..0ef40d6de1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthere\sis\sno\smention\sof\sa\stable\sin\ssqlite_stat1,\suse\sOP_Rewind\sto\ssee\sif\nthe\stable\sis\sempty\sprior\sto\sinvoking\sANALYZE. -D 2024-02-19T20:12:30.657 +C Fix\sa\sminor\shiccup\sin\sthe\scomputation\sof\sthe\snumber\sof\sbtrees\sto\sbe\sscanned. +D 2024-02-19T20:47:59.472 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c 280945eb4a7b78c82cd1b4d0bdefbb69af685a654f8fbfeb3bfd7dc8fede5fda +F src/pragma.c d0e7832bd99ea1c2695466894401f59fd4589ad98f93285a1d600877d0420070 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c d3392b2a20ee314ddeef34fb43c904bf4619eb20ff9a9e07e3950a7e4dcd6912 @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 636615358aebfef80a657d09f15f608df9e64882b2aaa5574ce31a4f1c71fd70 -R 98db8daafbcb183bee3a0d5c6d6f3dec +P 0cc93b19de597866292e0696f89fbd0e22a6cbb374678f5cc096fb889210fe3c +R d2d796f1af82f58905d69562436e5dd5 U drh -Z 53d562a05375d893c6e8245ccbe9b40a +Z d7a267289f8d852622ceaa2fe9f0686a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b4d235c325..71cf5e34f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cc93b19de597866292e0696f89fbd0e22a6cbb374678f5cc096fb889210fe3c \ No newline at end of file +91302d9b2720d6b5184e1c24db55d351544adfde4b3d2ddcdc1561f79f11985a \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index e65233389e..571d993714 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2529,7 +2529,6 @@ void sqlite3Pragma( hasStat1 = 1; }else{ szThreshold = -1; /* Always analyze if any index lacks statistics */ - break; } } From ae71fa5d59fa9da292770ec3d14b1f330c079e79 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 19 Feb 2024 23:58:26 +0000 Subject: [PATCH 11/15] Minor fixes. FossilOrigin-Name: ed5afebc410e3d430e2428d35f027bffb218e5f11036a42f97b080153e89016e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0ef40d6de1..007c09ab9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sminor\shiccup\sin\sthe\scomputation\sof\sthe\snumber\sof\sbtrees\sto\sbe\sscanned. -D 2024-02-19T20:47:59.472 +C Minor\sfixes. +D 2024-02-19T23:58:26.332 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c d0e7832bd99ea1c2695466894401f59fd4589ad98f93285a1d600877d0420070 +F src/pragma.c 613d27816d26e9db585fd4a34a627cfb436dc5b63edd73546b0fdaacdfa33772 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c d3392b2a20ee314ddeef34fb43c904bf4619eb20ff9a9e07e3950a7e4dcd6912 @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cc93b19de597866292e0696f89fbd0e22a6cbb374678f5cc096fb889210fe3c -R d2d796f1af82f58905d69562436e5dd5 +P 91302d9b2720d6b5184e1c24db55d351544adfde4b3d2ddcdc1561f79f11985a +R f5703f8a3a60803127fbc9cadd6a43fe U drh -Z d7a267289f8d852622ceaa2fe9f0686a +Z 926161dae17b79fd58ce2e39147c492e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 71cf5e34f0..51dda07214 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91302d9b2720d6b5184e1c24db55d351544adfde4b3d2ddcdc1561f79f11985a \ No newline at end of file +ed5afebc410e3d430e2428d35f027bffb218e5f11036a42f97b080153e89016e \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 571d993714..ca34e80af0 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2569,6 +2569,7 @@ void sqlite3Pragma( sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp2(v, OP_Rewind, iTabCur, sqlite3VdbeCurrentAddr(v)+2+(opMask&1)); + VdbeCoverage(v); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", db->aDb[iDb].zDbSName, pTab->zName); @@ -2587,10 +2588,10 @@ void sqlite3Pragma( /* In a schema with a large number of tables and indexes, scale back ** the analysis_limit to avoid excess run-time in the worst case. */ - if( !db->mallocFailed && nBtree>100 ){ + if( !db->mallocFailed && nLimit>0 && nBtree>100 ){ int iAddr, iEnd; VdbeOp *aOp; - int nLimit = 100*SQLITE_DEFAULT_OPTIMIZE_LIMIT/nBtree; + nLimit = 100*nLimit/nBtree; if( nLimit<100 ) nLimit = 100; aOp = sqlite3VdbeGetOp(v, 0); iEnd = sqlite3VdbeCurrentAddr(v); From 4189c44cff384341fb2a7780a7b5b35c734db9af Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Feb 2024 12:14:07 +0000 Subject: [PATCH 12/15] Another simplification of the PRAGMA optimize logic for improved coverage. FossilOrigin-Name: 6c5a0c85454e3c658e51fab611c169c034447174022eebc52fd8619b528a4765 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 007c09ab9b..e83a2aea80 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sfixes. -D 2024-02-19T23:58:26.332 +C Another\ssimplification\sof\sthe\sPRAGMA\soptimize\slogic\sfor\simproved\scoverage. +D 2024-02-20T12:14:07.075 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c 613d27816d26e9db585fd4a34a627cfb436dc5b63edd73546b0fdaacdfa33772 +F src/pragma.c cc9ccfde12caf4db782332dc7edd6c0aaf0bfa9bb8c0bff86a3f138521f11b27 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c d3392b2a20ee314ddeef34fb43c904bf4619eb20ff9a9e07e3950a7e4dcd6912 @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 91302d9b2720d6b5184e1c24db55d351544adfde4b3d2ddcdc1561f79f11985a -R f5703f8a3a60803127fbc9cadd6a43fe +P ed5afebc410e3d430e2428d35f027bffb218e5f11036a42f97b080153e89016e +R 1d7e6aefb784a3b060fc17ebc00f9588 U drh -Z 926161dae17b79fd58ce2e39147c492e +Z 39fd458c30bb322f77d1633b20055bca # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 51dda07214..a30b6b2081 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed5afebc410e3d430e2428d35f027bffb218e5f11036a42f97b080153e89016e \ No newline at end of file +6c5a0c85454e3c658e51fab611c169c034447174022eebc52fd8619b528a4765 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index ca34e80af0..877fb3f721 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2557,16 +2557,15 @@ void sqlite3Pragma( /* Reanalyze if the table is 10 times larger or smaller than ** the last analysis. Unconditional reanalysis if there are ** unanalyzed indexes. */ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); if( szThreshold>=0 ){ const LogEst iRange = 33; /* 10x size change */ - sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold>=iRange ? szThreshold-iRange : -1, szThreshold+iRange); VdbeCoverage(v); - }else if( !hasStat1 ){ - sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + }else{ sqlite3VdbeAddOp2(v, OP_Rewind, iTabCur, sqlite3VdbeCurrentAddr(v)+2+(opMask&1)); VdbeCoverage(v); From 4b2eeb289bef1735ab56aa4867f2c3b69550b9c2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Feb 2024 13:10:46 +0000 Subject: [PATCH 13/15] Remove an unused variable from the merge in the previous check-in. FossilOrigin-Name: 5c26bf379e18e28d800525bd03e2d5597e2f37674979302f351b99abfc064749 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/pragma.c | 6 +----- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index c071235824..abca404cf5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhancements\sto\sPRAGMA\soptimize\sand\sANALYZE.\s\sAdd\sthe\s0x10000\sflag\sto\nPRAGMA\soptimize.\s\sANALYZE\snow\srecords\szero-size\spartial\sindexes\sin\sthe\nsqlite_stat1\stable.\s\sPRAGMA\soptimize\slooks\sfor\sboth\sgrowth\sand\sshrinkage\nin\stable\ssizes,\sand\suses\stighter\sbounds\s(10x\srather\sthan\s25x)\sto\strigger\na\sre-analyze.\s\sPRAGMA\soptimize\sautomatically\suses\sare\sreasonable\nanalysis_limit\sto\sprevent\sexcessive\sruntimes. -D 2024-02-20T12:48:00.737 +C Remove\san\sunused\svariable\sfrom\sthe\smerge\sin\sthe\sprevious\scheck-in. +D 2024-02-20T13:10:46.615 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c cc9ccfde12caf4db782332dc7edd6c0aaf0bfa9bb8c0bff86a3f138521f11b27 +F src/pragma.c f35ba8fb0b3782ef671cc38b3cdaa457e82befe022558f343240344e57d5f5f4 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c d3392b2a20ee314ddeef34fb43c904bf4619eb20ff9a9e07e3950a7e4dcd6912 @@ -2162,9 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f18b2524da6bbbcf372b292df52fbe6efa49fd6e1f1f13ef2447279d559f7a08 6c5a0c85454e3c658e51fab611c169c034447174022eebc52fd8619b528a4765 -R 1d7e6aefb784a3b060fc17ebc00f9588 -T +closed 6c5a0c85454e3c658e51fab611c169c034447174022eebc52fd8619b528a4765 +P 63ef234e88857a653fa3541e80d59802ceccb806ac8296e8bae79a385b7086f7 +R a21cfe7fc11d4cc3af701137f92d89ba U drh -Z b62a3cf41afb08b6428ca7c918282668 +Z d6568f521192ee23916aac9c92a235a6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 54028cc0d8..262b24e277 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63ef234e88857a653fa3541e80d59802ceccb806ac8296e8bae79a385b7086f7 \ No newline at end of file +5c26bf379e18e28d800525bd03e2d5597e2f37674979302f351b99abfc064749 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 877fb3f721..af51d18657 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2486,7 +2486,6 @@ void sqlite3Pragma( int nCheck = 0; /* Number of tables to be optimized */ int nBtree = 0; /* Number of btrees to scan */ int nIndex; /* Number of indexes on the current table */ - int hasStat1; /* True if any STAT1 info available for the table */ if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -2521,13 +2520,10 @@ void sqlite3Pragma( ** indicate a new, unanalyzed index */ szThreshold = pTab->nRowLogEst; - hasStat1 = (pTab->tabFlags & TF_HasStat1)!=0; nIndex = 0; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ nIndex++; - if( pIdx->hasStat1 ){ - hasStat1 = 1; - }else{ + if( !pIdx->hasStat1 ){ szThreshold = -1; /* Always analyze if any index lacks statistics */ } } From b468e6832288292695ff5dc6782dd1e5334ee8f6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Feb 2024 13:11:27 +0000 Subject: [PATCH 14/15] Remove an unreachable branch from printf(). Change it into an assert(). FossilOrigin-Name: 4c9886b692706723f4b68f58dfef07ab19b192fd19b953917f23b55655af3e02 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index abca404cf5..40152b32a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\svariable\sfrom\sthe\smerge\sin\sthe\sprevious\scheck-in. -D 2024-02-20T13:10:46.615 +C Remove\san\sunreachable\sbranch\sfrom\sprintf().\s\sChange\sit\sinto\san\sassert(). +D 2024-02-20T13:11:27.626 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -735,7 +735,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 F src/pragma.c f35ba8fb0b3782ef671cc38b3cdaa457e82befe022558f343240344e57d5f5f4 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c -F src/printf.c d3392b2a20ee314ddeef34fb43c904bf4619eb20ff9a9e07e3950a7e4dcd6912 +F src/printf.c 10e8bad30042f8bd6114a013b4afc229ec8ad255ab27518d7d9f52e8cbc5cd0a F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d77c6160bc8f249c2196fdd3e75f66a1dd70e37aa25c39aedc7b1f93c42b7c6d F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 63ef234e88857a653fa3541e80d59802ceccb806ac8296e8bae79a385b7086f7 -R a21cfe7fc11d4cc3af701137f92d89ba +P 5c26bf379e18e28d800525bd03e2d5597e2f37674979302f351b99abfc064749 +R 521cc3374b233ca8db7b57619e6235d0 U drh -Z d6568f521192ee23916aac9c92a235a6 +Z f1be525f7ee6098da75f01bd7c5e0ac8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 262b24e277..a714b19d22 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c26bf379e18e28d800525bd03e2d5597e2f37674979302f351b99abfc064749 \ No newline at end of file +4c9886b692706723f4b68f58dfef07ab19b192fd19b953917f23b55655af3e02 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 2e09431bf2..186e95bb85 100644 --- a/src/printf.c +++ b/src/printf.c @@ -534,13 +534,14 @@ void sqlite3_str_vappendf( } exp = s.iDP-1; - if( xtype==etGENERIC && precision>0 ) precision--; /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ if( xtype==etGENERIC ){ + assert( precision>0 ); + precision--; flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ xtype = etEXP; From 21eda699d7154c045c72e8c848ebe2ade011165f Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 20 Feb 2024 15:38:36 +0000 Subject: [PATCH 15/15] Fixes to comments associated with the recent PRAGMA optimize enhancements. No changes to code. FossilOrigin-Name: 27a2113d78b35e324e9aedda7403c96c56ad0bed8c6b139fc5a179e8800b9109 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 5 +---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 40152b32a7..8ad3679fb6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\sbranch\sfrom\sprintf().\s\sChange\sit\sinto\san\sassert(). -D 2024-02-20T13:11:27.626 +C Fixes\sto\scomments\sassociated\swith\sthe\srecent\sPRAGMA\soptimize\senhancements.\nNo\schanges\sto\scode. +D 2024-02-20T15:38:36.661 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -732,7 +732,7 @@ F src/parse.y bfd6da46fc895cd8237400ff485d04ab0b32e47eb56de20982bb7f53e56c1f42 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c f35ba8fb0b3782ef671cc38b3cdaa457e82befe022558f343240344e57d5f5f4 +F src/pragma.c f5cb82c15b7455d42e748cad8a6c367db18ada0fadc9b4adba03de953a81d46a F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 10e8bad30042f8bd6114a013b4afc229ec8ad255ab27518d7d9f52e8cbc5cd0a @@ -2162,8 +2162,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5c26bf379e18e28d800525bd03e2d5597e2f37674979302f351b99abfc064749 -R 521cc3374b233ca8db7b57619e6235d0 +P 4c9886b692706723f4b68f58dfef07ab19b192fd19b953917f23b55655af3e02 +R 40f50207938b86842b8686e871138379 U drh -Z f1be525f7ee6098da75f01bd7c5e0ac8 +Z 751534f3f8e8461179e5c9344b944b7a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a714b19d22..a82c127f34 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c9886b692706723f4b68f58dfef07ab19b192fd19b953917f23b55655af3e02 \ No newline at end of file +27a2113d78b35e324e9aedda7403c96c56ad0bed8c6b139fc5a179e8800b9109 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index af51d18657..62fd17911a 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2427,9 +2427,6 @@ void sqlite3Pragma( ** database will usually be less than 100 milliseconds on ** a RaspberryPI-4 class machine. On by default. ** - ** 0x00020 Run ANALYZE on any table that has a index that lacks an - ** entry in the sqlite_stat1 table. On by default. - ** ** 0x10000 Look at tables to see if they need to be reanalyzed ** due to growth or shrinkage even if they have not been ** queried during the current connection. Off by default. @@ -2456,7 +2453,7 @@ void sqlite3Pragma( ** (4b) One or more indexes on the table lacks an entry ** in the sqlite_stat1 table. ** (4c) The query planner used sqlite_stat1-style statistics for one - ** or more indexes of the tableat some point during the lifetime + ** or more indexes of the table at some point during the lifetime ** of the current connection. ** ** (5) One or more of the following is true: