From a748fdcc43a2f4a5f59acb6c30742d958b2dcac2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Mar 2012 01:34:47 +0000 Subject: [PATCH 1/5] Evaluate typeof(X) and length(Y) where X is any column and Y is a blob column without actually loading X and Y from disk. FossilOrigin-Name: b899dbeb60752843287e2c6ad3577e1d00f0d587 --- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- src/delete.c | 2 +- src/expr.c | 38 ++++++++++++++++++++++++++------------ src/func.c | 10 ++++------ src/select.c | 2 +- src/sqliteInt.h | 24 +++++++++++++++++------- src/vdbe.c | 27 ++++++++++++++++++++++----- src/where.c | 2 +- test/func.test | 33 +++++++++++++++++++++++++++++++++ test/pager1.test | 31 ++++++++++++++++++++++++++++--- 11 files changed, 148 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 87409f9e29..c78d6a8b74 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.7.12 -D 2012-03-25T17:25:38.638 +C Evaluate\stypeof(X)\sand\slength(Y)\swhere\sX\sis\sany\scolumn\sand\sY\sis\sa\sblob\scolumn\nwithout\sactually\sloading\sX\sand\sY\sfrom\sdisk. +D 2012-03-28T01:34:47.425 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,11 +133,11 @@ F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c a9c26822515f81ec21588cbb482ca6724be02e33 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 -F src/delete.c 51d32f0a9c880663e54ce309f52e40c325d5e112 -F src/expr.c 00675123e0beec98f999aa4594d2cbe1fec33c1b +F src/delete.c 4c20ea4f6213b3bc1c6a510586864b679946e05e +F src/expr.c 86711d201e6ee0e795571900edf20722d008e3ab F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 657212460bf5cfd3ae607d12ea62092844c227b5 -F src/func.c e75f41c421f00762ab9da7dc8fb90af3972cf99f +F src/func.c c6b3c94320253a35bda43fb69cc292618e3285d6 F src/global.c 4cfdca5cb0edd33c4d021baec4ede958cb2c793b F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 @@ -180,11 +180,11 @@ F src/printf.c 7ffb4ebb8b341f67e049695ba031da717b3d2699 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 3d3e80a98f203ac6b9329e9621e29eda85ddfd40 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 44ccdcb5d2a1c48622c179b2d72167b716388581 +F src/select.c f6f141cb1ea13f1e6564d3e162700e4937baa2a1 F src/shell.c 3179db5d4ff33d62d59a024dbfd2a116390ef7b0 F src/sqlite.h.in 11a883919b0baf4ffaa7550cfeef99be613ec2bf F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h e65429a6f19b41720561b9434b2192574a91cfa2 +F src/sqliteInt.h 3756ece33f1e7f8fe2adf8e523566825c809316e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -240,7 +240,7 @@ F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 4f6cfad661b2e3454b0cdd5b1b9d39a54942d0e3 F src/vacuum.c bfd53f9bd20a8fdb70b0fa8e77182b866875c0d8 -F src/vdbe.c 32720e873ed0a23e6ee928b676cd995864b984d6 +F src/vdbe.c b91a9135fa4ecd73805cfda7b051dbcc155a0fb8 F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 F src/vdbeapi.c 3662b6a468a2a4605a15dfab313baa6dff81ad91 @@ -253,7 +253,7 @@ F src/vtab.c ab90fb600a3f5e4b7c48d22a4cdb2d6b23239847 F src/wal.c 7bb3ad807afc7973406c805d5157ec7a2f65e146 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 6baab5dfcf4472552c0346d04f6fd2f4f8539c78 +F src/where.c 44d78f5811594065ebbb5354da7f9e6b8b1306d6 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -497,7 +497,7 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 2081c357bb6f170f34ef8e08c6abb88002b95c69 -F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca +F test/func.test 6966ad939b8fccc7d48d18e0c1fc8cd1a9f197e6 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz.test 77fd50afc12847af50fcf1941679d90adebadde6 @@ -626,7 +626,7 @@ F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 -F test/pager1.test cf8f40cf77b5c4f762b1e8492390d61b46a81623 +F test/pager1.test eb6d64d2e148dc4bfc6420605bee3717e2d5f10c F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 30b8dd326d28c0c08543989e376011ea41773a7e -R 9c26fe2aaab0d138abb3ef6a21a226f0 +P d95f9fb713c7ba4e570556d835fbd77e574afdea +R 89de5eb864ab0dd31529150cd7eba40a U drh -Z 54613266ba9e6cb2a27d5ad8392e74a5 +Z 999e726acb9e2120206bfa344ce7af80 diff --git a/manifest.uuid b/manifest.uuid index 5306c9df0b..e1ec8508a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d95f9fb713c7ba4e570556d835fbd77e574afdea \ No newline at end of file +b899dbeb60752843287e2c6ad3577e1d00f0d587 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index f666b90c83..eead4856b1 100644 --- a/src/delete.c +++ b/src/delete.c @@ -374,7 +374,7 @@ void sqlite3DeleteFrom( pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK ); if( pWInfo==0 ) goto delete_from_cleanup; - regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); + regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid, 0); sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); if( db->flags & SQLITE_CountRows ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); diff --git a/src/expr.c b/src/expr.c index 79dd8f4961..b76a8afe12 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2032,15 +2032,6 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ */ #ifndef NDEBUG for(i=0, p=pParse->aColCache; iiReg && p->iTable==iTab && p->iColumn==iCol ){ - cacheEntryClear(pParse, p); - p->iLevel = pParse->iCacheLevel; - p->iReg = iReg; - p->lru = pParse->iCacheCnt++; - return; - } -#endif assert( p->iReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); } #endif @@ -2175,7 +2166,8 @@ int sqlite3ExprCodeGetColumn( Table *pTab, /* Description of the table we are reading from */ int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ + int iReg, /* Store results here */ + u8 p5 /* P5 value for OP_Column */ ){ Vdbe *v = pParse->pVdbe; int i; @@ -2190,7 +2182,11 @@ int sqlite3ExprCodeGetColumn( } assert( v!=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); - sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + if( p5 ){ + sqlite3VdbeChangeP5(v, p5); + }else{ + sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + } return iReg; } @@ -2318,7 +2314,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pExpr->iColumn + pParse->ckBase; }else{ inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, - pExpr->iColumn, pExpr->iTable, target); + pExpr->iColumn, pExpr->iTable, target, + pExpr->op2); } break; } @@ -2595,6 +2592,23 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( pFarg ){ r1 = sqlite3GetTempRange(pParse, nFarg); + + /* For length() and typeof() functions with a column argument, + ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG + ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data + ** loading. + */ + if( (pDef->flags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + assert( nFarg==1 ); + assert( pFarg->a[0].pExpr!=0 ); + if( pFarg->a[0].pExpr->op==TK_COLUMN ){ + assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); + assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); + testcase( pDef->flags==SQLITE_FUNC_LENGTH ); + pFarg->a[0].pExpr->op2 = pDef->flags; + } + } + sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ sqlite3ExprCodeExprList(pParse, pFarg, r1, 1); sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */ diff --git a/src/func.c b/src/func.c index c66ad28aba..6ffc7184b0 100644 --- a/src/func.c +++ b/src/func.c @@ -1542,8 +1542,8 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), - FUNCTION(typeof, 1, 0, 0, typeofFunc ), - FUNCTION(length, 1, 0, 0, lengthFunc ), + FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), + FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), @@ -1555,11 +1555,9 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), -/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ - {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, + FUNCTION2(coalesce, -1, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), -/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ - {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, + FUNCTION2(ifnull, 2, 0, 0, ifnullFunc, SQLITE_FUNC_COALESCE), FUNCTION(random, 0, 0, 0, randomFunc ), FUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), diff --git a/src/select.c b/src/select.c index 3efe014d8b..c225013672 100644 --- a/src/select.c +++ b/src/select.c @@ -4228,7 +4228,7 @@ int sqlite3Select( int r2; r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); + pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); if( r1!=r2 ){ sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 15c3f302e2..deb576d2d7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1009,14 +1009,18 @@ struct FuncDestructor { }; /* -** Possible values for FuncDef.flags +** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There +** are assert() statements in the code to verify this. */ #define SQLITE_FUNC_LIKE 0x01 /* Candidate for the LIKE optimization */ #define SQLITE_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */ #define SQLITE_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */ #define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_COUNT 0x20 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_COUNT 0x10 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x20 /* Built-in coalesce() or ifnull() function */ +#define SQLITE_FUNC_LENGTH 0x40 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x80 /* Built-in typeof() function */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1044,7 +1048,10 @@ struct FuncDestructor { ** parameter. */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ + {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ + {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ @@ -1667,6 +1674,7 @@ struct Expr { i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ u8 flags2; /* Second set of flags. EP2_... */ u8 op2; /* If a TK_REGISTER, the original value of Expr.op */ + /* If TK_COLUMN, the value of p5 for OP_Column */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ Table *pTab; /* Table for TK_COLUMN expressions. */ #if SQLITE_MAX_EXPR_DEPTH>0 @@ -1689,7 +1697,7 @@ struct Expr { #define EP_FixedDest 0x0200 /* Result needed in a specific register */ #define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Hint 0x1000 /* Optimizer hint. Not required for correctness */ +#define EP_Hint 0x1000 /* Not used */ #define EP_Reduced 0x2000 /* Expr struct is EXPR_REDUCEDSIZE bytes only */ #define EP_TokenOnly 0x4000 /* Expr struct is EXPR_TOKENONLYSIZE bytes only */ #define EP_Static 0x8000 /* Held in memory not obtained from malloc() */ @@ -2276,7 +2284,7 @@ struct AuthContext { }; /* -** Bitfield flags for P5 value in OP_Insert and OP_Delete +** Bitfield flags for P5 value in various opcodes. */ #define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ @@ -2284,6 +2292,8 @@ struct AuthContext { #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ #define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ +#define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ +#define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ /* * Each trigger present in the database schema is stored as an instance of @@ -2767,7 +2777,7 @@ void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); void sqlite3WhereEnd(WhereInfo*); -int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int); +int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); void sqlite3ExprCodeCopy(Parse*, int, int, int); diff --git a/src/vdbe.c b/src/vdbe.c index 749830bc0d..93032936ae 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2127,6 +2127,11 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** then the cache of the cursor is reset prior to extracting the column. ** The first OP_Column against a pseudo-table after the value of the content ** register has changed should have this bit set. +** +** If the OPFLAG_LENGTHARG bit is set on P5 then the result is guaranteed +** to only be used as the argument of a length() or typeof() function and +** so loading of large blobs and strings can be skipped - all that is necessary +** is that the size and type information be set. */ case OP_Column: { u32 payloadSize; /* Number of bytes in the record */ @@ -2380,12 +2385,24 @@ case OP_Column: { sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest); }else{ len = sqlite3VdbeSerialTypeLen(aType[p2]); - sqlite3VdbeMemMove(&sMem, pDest); - rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, &sMem); - if( rc!=SQLITE_OK ){ - goto op_column_out; + if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 + && (((t = aType[p2])>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) + ){ + /* Content is irrelevant for the typeof() function and for + ** the length(x) function is x is a blob. So we might as well use + ** bogus content rather than reading content from disk. NULL works + ** for text and blob and whatever is in the payloadSize64 variable + ** will work for everything else. */ + zData = t<12 ? (char*)&payloadSize64 : 0; + }else{ + sqlite3VdbeMemMove(&sMem, pDest); + rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, + &sMem); + if( rc!=SQLITE_OK ){ + goto op_column_out; + } + zData = sMem.z; } - zData = sMem.z; sqlite3VdbeSerialGet((u8*)zData, aType[p2], pDest); } pDest->enc = encoding; diff --git a/src/where.c b/src/where.c index e6083f0033..2f3dc77e12 100644 --- a/src/where.c +++ b/src/where.c @@ -4383,7 +4383,7 @@ static Bitmask codeOneLoopStart( int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); int r; r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, - regRowid); + regRowid, 0); sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, sqlite3VdbeCurrentAddr(v)+2, r, iSet); } diff --git a/test/func.test b/test/func.test index eef0543986..37631a419f 100644 --- a/test/func.test +++ b/test/func.test @@ -1247,4 +1247,37 @@ do_test func-28.1 { } } {1 {unknown function: nosuchfunc()}} +# Verify that the length() and typeof() functions do not actually load +# the content of their argument. +# +do_test func-29.1 { + db eval { + CREATE TABLE t29(id INTEGER PRIMARY KEY, x, y); + INSERT INTO t29 VALUES(1, 2, 3), (2, NULL, 4), (3, 4.5, 5); + INSERT INTO t29 VALUES(4, randomblob(1000000), 6); + INSERT INTO t29 VALUES(5, "hello", 7); + } + db close + sqlite3 db test.db + sqlite3_db_status db CACHE_MISS 1 + db eval {SELECT typeof(x), length(x), typeof(y) FROM t29 ORDER BY id} +} {integer 1 integer null {} integer real 3 integer blob 1000000 integer text 5 integer} +do_test func-29.2 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x<5} {set x 1} + set x +} {1} +do_test func-29.3 { + db close + sqlite3 db test.db + sqlite3_db_status db CACHE_MISS 1 + db eval {SELECT typeof(+x) FROM t29 ORDER BY id} +} {integer null real blob text} +do_test func-29.4 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x>100} {set x many} + set x +} {many} + + finish_test diff --git a/test/pager1.test b/test/pager1.test index 45253857c3..2ac56dec8c 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -1768,7 +1768,7 @@ do_test pager1-18.2 { catchsql { SELECT count(*) FROM t1 } db2 } {1 {database disk image is malformed}} db2 close -do_test pager1-18.3 { +do_test pager1-18.3.1 { execsql { CREATE TABLE t2(x); INSERT INTO t2 VALUES(a_string(5000)); @@ -1776,13 +1776,38 @@ do_test pager1-18.3 { set pgno [expr ([file size test.db] / 1024)-2] hexio_write test.db [expr ($pgno-1)*1024] 00000000 sqlite3 db2 test.db - catchsql { SELECT length(x) FROM t2 } db2 + # even though x is malformed, because typeof() does + # not load the content of x, the error is not noticed. + catchsql { SELECT typeof(x) FROM t2 } db2 +} {0 text} +do_test pager1-18.3.2 { + # in this case, the value of x is loaded and so the error is + # detected + catchsql { SELECT length(x||'') FROM t2 } db2 +} {1 {database disk image is malformed}} +db2 close +do_test pager1-18.3.3 { + execsql { + DELETE FROM t2; + INSERT INTO t2 VALUES(randomblob(5000)); + } + set pgno [expr ([file size test.db] / 1024)-2] + hexio_write test.db [expr ($pgno-1)*1024] 00000000 + sqlite3 db2 test.db + # even though x is malformed, because length() and typeof() do + # not load the content of x, the error is not noticed. + catchsql { SELECT length(x), typeof(x) FROM t2 } db2 +} {0 {5000 blob}} +do_test pager1-18.3.4 { + # in this case, the value of x is loaded and so the error is + # detected + catchsql { SELECT length(x||'') FROM t2 } db2 } {1 {database disk image is malformed}} db2 close do_test pager1-18.4 { hexio_write test.db [expr ($pgno-1)*1024] 90000000 sqlite3 db2 test.db - catchsql { SELECT length(x) FROM t2 } db2 + catchsql { SELECT length(x||'') FROM t2 } db2 } {1 {database disk image is malformed}} db2 close do_test pager1-18.5 { From 31ec740a3dfc1190ed972fba8cb99e4de7db0254 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Mar 2012 02:43:20 +0000 Subject: [PATCH 2/5] Fix the typeof() and length() optimization so that it works for aggregates as well as scalar queries. FossilOrigin-Name: bc18215a8a660442db6ddeeda4a88df0acffe0f7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c78d6a8b74..83178b14b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Evaluate\stypeof(X)\sand\slength(Y)\swhere\sX\sis\sany\scolumn\sand\sY\sis\sa\sblob\scolumn\nwithout\sactually\sloading\sX\sand\sY\sfrom\sdisk. -D 2012-03-28T01:34:47.425 +C Fix\sthe\stypeof()\sand\slength()\soptimization\sso\sthat\sit\sworks\sfor\saggregates\nas\swell\sas\sscalar\squeries. +D 2012-03-28T02:43:20.877 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -134,7 +134,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c a9c26822515f81ec21588cbb482ca6724be02e33 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c 4c20ea4f6213b3bc1c6a510586864b679946e05e -F src/expr.c 86711d201e6ee0e795571900edf20722d008e3ab +F src/expr.c 1c351f385950a40b0283328eb925641a3aa50a84 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 657212460bf5cfd3ae607d12ea62092844c227b5 F src/func.c c6b3c94320253a35bda43fb69cc292618e3285d6 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P d95f9fb713c7ba4e570556d835fbd77e574afdea -R 89de5eb864ab0dd31529150cd7eba40a +P b899dbeb60752843287e2c6ad3577e1d00f0d587 +R f4c337d261ff97dcd84dc2dd2a1e0095 U drh -Z 999e726acb9e2120206bfa344ce7af80 +Z 615304a5ba3b27e6b38911664d03f093 diff --git a/manifest.uuid b/manifest.uuid index e1ec8508a4..93fa65d24b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b899dbeb60752843287e2c6ad3577e1d00f0d587 \ No newline at end of file +bc18215a8a660442db6ddeeda4a88df0acffe0f7 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index b76a8afe12..1915454742 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2599,9 +2599,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ** loading. */ if( (pDef->flags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + u8 op; assert( nFarg==1 ); assert( pFarg->a[0].pExpr!=0 ); - if( pFarg->a[0].pExpr->op==TK_COLUMN ){ + op = pFarg->a[0].pExpr->op; + if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); testcase( pDef->flags==SQLITE_FUNC_LENGTH ); From 3c888b7d1b39f1e08bc2cf627ec7b4ec3b98b63e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Mar 2012 02:51:51 +0000 Subject: [PATCH 3/5] Test cases for length() of a large blob in an aggregate query. FossilOrigin-Name: d095fa4bfabd765c8e935ed227a334161097dd34 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/func.test | 11 +++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 83178b14b7..398f5477a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\stypeof()\sand\slength()\soptimization\sso\sthat\sit\sworks\sfor\saggregates\nas\swell\sas\sscalar\squeries. -D 2012-03-28T02:43:20.877 +C Test\scases\sfor\slength()\sof\sa\slarge\sblob\sin\san\saggregate\squery. +D 2012-03-28T02:51:51.807 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -497,7 +497,7 @@ F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 2081c357bb6f170f34ef8e08c6abb88002b95c69 -F test/func.test 6966ad939b8fccc7d48d18e0c1fc8cd1a9f197e6 +F test/func.test 9809b7622d721904a8cc33c1ffb87f46d506ed01 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz.test 77fd50afc12847af50fcf1941679d90adebadde6 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P b899dbeb60752843287e2c6ad3577e1d00f0d587 -R f4c337d261ff97dcd84dc2dd2a1e0095 +P bc18215a8a660442db6ddeeda4a88df0acffe0f7 +R 3ecaca7bd152423dde5d0467891373c7 U drh -Z 615304a5ba3b27e6b38911664d03f093 +Z c57e0152e24243ecb61f7c7f2f11b01f diff --git a/manifest.uuid b/manifest.uuid index 93fa65d24b..2f703b38c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bc18215a8a660442db6ddeeda4a88df0acffe0f7 \ No newline at end of file +d095fa4bfabd765c8e935ed227a334161097dd34 \ No newline at end of file diff --git a/test/func.test b/test/func.test index 37631a419f..ba1ea026d7 100644 --- a/test/func.test +++ b/test/func.test @@ -1278,6 +1278,17 @@ do_test func-29.4 { if {$x>100} {set x many} set x } {many} +do_test func-29.5 { + db close + sqlite3 db test.db + sqlite3_db_status db CACHE_MISS 1 + db eval {SELECT sum(length(x)) FROM t29} +} {1000009} +do_test func-29.6 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x<5} {set x 1} + set x +} {1} finish_test From dda5c08bf629e982318eac23dae51827fd39d391 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Mar 2012 13:41:10 +0000 Subject: [PATCH 4/5] Improvements to comments. Minor changes to code in the hot path of OP_Column - with the hope of get a few cycles of performance improvement. FossilOrigin-Name: ca093103437f141caa3eb11539c6eb7b4dd65175 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 25 +++++++++++++------------ 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 398f5477a1..d2e78d5c20 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scases\sfor\slength()\sof\sa\slarge\sblob\sin\san\saggregate\squery. -D 2012-03-28T02:51:51.807 +C Improvements\sto\scomments.\s\sMinor\schanges\sto\scode\sin\sthe\shot\spath\sof\nOP_Column\s-\swith\sthe\shope\sof\sget\sa\sfew\scycles\sof\sperformance\simprovement. +D 2012-03-28T13:41:10.884 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -240,7 +240,7 @@ F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 4f6cfad661b2e3454b0cdd5b1b9d39a54942d0e3 F src/vacuum.c bfd53f9bd20a8fdb70b0fa8e77182b866875c0d8 -F src/vdbe.c b91a9135fa4ecd73805cfda7b051dbcc155a0fb8 +F src/vdbe.c 807532457744a264d19e929b2075dbec90d2a36d F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 F src/vdbeapi.c 3662b6a468a2a4605a15dfab313baa6dff81ad91 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P bc18215a8a660442db6ddeeda4a88df0acffe0f7 -R 3ecaca7bd152423dde5d0467891373c7 +P d095fa4bfabd765c8e935ed227a334161097dd34 +R 78ef170f6dad0ba0a0f49e941cd533cb U drh -Z c57e0152e24243ecb61f7c7f2f11b01f +Z 6f108b9c2dd42e83ba81e7472cbc6ad3 diff --git a/manifest.uuid b/manifest.uuid index 2f703b38c1..3d77c17c7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d095fa4bfabd765c8e935ed227a334161097dd34 \ No newline at end of file +ca093103437f141caa3eb11539c6eb7b4dd65175 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 93032936ae..fa8dcdf1b7 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2128,10 +2128,10 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** The first OP_Column against a pseudo-table after the value of the content ** register has changed should have this bit set. ** -** If the OPFLAG_LENGTHARG bit is set on P5 then the result is guaranteed -** to only be used as the argument of a length() or typeof() function and -** so loading of large blobs and strings can be skipped - all that is necessary -** is that the size and type information be set. +** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when +** the result is guaranteed to only be used as the argument of a length() +** or typeof() function, respectively. The loading of large blobs can be +** skipped for length() and all content loading can be skipped for typeof(). */ case OP_Column: { u32 payloadSize; /* Number of bytes in the record */ @@ -2272,7 +2272,7 @@ case OP_Column: { pC->aRow = 0; } } - /* The following assert is true in all cases accept when + /* The following assert is true in all cases except when ** the database file has been corrupted externally. ** assert( zRec!=0 || avail>=payloadSize || avail>=9 ); */ szHdr = getVarint32((u8*)zData, offset); @@ -2347,11 +2347,11 @@ case OP_Column: { break; } }else{ - /* If i is less that nField, then there are less fields in this + /* If i is less that nField, then there are fewer fields in this ** record than SetNumColumns indicated there are columns in the ** table. Set the offset for any extra columns not present in - ** the record to 0. This tells code below to store a NULL - ** instead of deserializing a value from the record. + ** the record to 0. This tells code below to store the default value + ** for the column instead of deserializing a value from the record. */ aOffset[i] = 0; } @@ -2384,12 +2384,13 @@ case OP_Column: { VdbeMemRelease(pDest); sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest); }else{ - len = sqlite3VdbeSerialTypeLen(aType[p2]); + t = aType[p2]; + len = sqlite3VdbeSerialTypeLen(t); if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 - && (((t = aType[p2])>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) + && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) ){ /* Content is irrelevant for the typeof() function and for - ** the length(x) function is x is a blob. So we might as well use + ** the length(X) function if X is a blob. So we might as well use ** bogus content rather than reading content from disk. NULL works ** for text and blob and whatever is in the payloadSize64 variable ** will work for everything else. */ @@ -2403,7 +2404,7 @@ case OP_Column: { } zData = sMem.z; } - sqlite3VdbeSerialGet((u8*)zData, aType[p2], pDest); + sqlite3VdbeSerialGet((u8*)zData, t, pDest); } pDest->enc = encoding; }else{ From ac5e749be8997bd2d216eccd675ebe7ddbd2a888 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Mar 2012 16:14:50 +0000 Subject: [PATCH 5/5] Minor changes to the core of OP_Column for performance and to clarify the critical path. FossilOrigin-Name: 868394761e41b7483a5874426ee052dfb3a9e2be --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d2e78d5c20..36e1985ccd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\scomments.\s\sMinor\schanges\sto\scode\sin\sthe\shot\spath\sof\nOP_Column\s-\swith\sthe\shope\sof\sget\sa\sfew\scycles\sof\sperformance\simprovement. -D 2012-03-28T13:41:10.884 +C Minor\schanges\sto\sthe\score\sof\sOP_Column\sfor\sperformance\sand\sto\sclarify\sthe\ncritical\spath. +D 2012-03-28T16:14:50.602 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -240,7 +240,7 @@ F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 4f6cfad661b2e3454b0cdd5b1b9d39a54942d0e3 F src/vacuum.c bfd53f9bd20a8fdb70b0fa8e77182b866875c0d8 -F src/vdbe.c 807532457744a264d19e929b2075dbec90d2a36d +F src/vdbe.c 8913926230bfc9d183fcd44e3d023c6d393b6548 F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 F src/vdbeapi.c 3662b6a468a2a4605a15dfab313baa6dff81ad91 @@ -993,7 +993,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P d095fa4bfabd765c8e935ed227a334161097dd34 -R 78ef170f6dad0ba0a0f49e941cd533cb +P ca093103437f141caa3eb11539c6eb7b4dd65175 +R 14c36b139bb9a241f9fc5fc1fa96e28f U drh -Z 6f108b9c2dd42e83ba81e7472cbc6ad3 +Z 95f518d28e036a56a444945b5630fdfa diff --git a/manifest.uuid b/manifest.uuid index 3d77c17c7f..d80571e66d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca093103437f141caa3eb11539c6eb7b4dd65175 \ No newline at end of file +868394761e41b7483a5874426ee052dfb3a9e2be \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index fa8dcdf1b7..9daa9cbff6 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2381,11 +2381,12 @@ case OP_Column: { if( aOffset[p2] ){ assert( rc==SQLITE_OK ); if( zRec ){ + /* This is the common case where the whole row fits on a single page */ VdbeMemRelease(pDest); sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest); }else{ + /* This branch happens only when the row overflows onto multiple pages */ t = aType[p2]; - len = sqlite3VdbeSerialTypeLen(t); if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0) ){ @@ -2396,6 +2397,7 @@ case OP_Column: { ** will work for everything else. */ zData = t<12 ? (char*)&payloadSize64 : 0; }else{ + len = sqlite3VdbeSerialTypeLen(t); sqlite3VdbeMemMove(&sMem, pDest); rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, &sMem);