Merge the latest enhancements from trunk.

FossilOrigin-Name: bd57e6d923d3b04f0a07aaf18bf389d2b2b7efc7c57e8cb37e6ef910662d8397

ext/fts3/fts3_tokenizer.c

@@ -481,7 +481,7 @@ int sqlite3Fts3InitHashTable(
 ){
   int rc = SQLITE_OK;
   void *p = (void *)pHash;
-  const int any = SQLITE_ANY;
+  const int any = SQLITE_ANY|SQLITE_DIRECTONLY;
 
 #ifdef SQLITE_TEST
   char *zTest = 0;

ext/fts5/test/fts5integrity.test

@@ -255,4 +255,31 @@ do_execsql_test 8.1 {
   INSERT INTO vt0(vt0, rank) VALUES('usermerge', 2);
 }
 
+#-------------------------------------------------------------------------
+# Ticket [771fe617]
+#
+reset_db
+do_execsql_test 9.0 {
+  PRAGMA encoding = 'UTF16';
+  CREATE VIRTUAL TABLE vt0 USING fts5(c0);
+}
+
+#explain_i { SELECT quote(SUBSTR(x'37', 0)); }
+#execsql { PRAGMA vdbe_trace = 1 }
+do_execsql_test 9.1.1 {
+  SELECT quote(SUBSTR(x'37', 0));
+} {X'37'}
+do_execsql_test 9.1.2 {
+  SELECT quote(x'37');
+} {X'37'}
+
+breakpoint
+do_execsql_test 9.2 {
+  INSERT INTO vt0 VALUES (SUBSTR(x'37', 0));
+--  INSERT INTO vt0 VALUES (x'37');
+}
+do_execsql_test 9.3 {
+  INSERT INTO vt0(vt0) VALUES('integrity-check');
+}
+
 finish_test

ext/misc/zipfile.c

@@ -1444,6 +1444,7 @@ static int zipfileBegin(sqlite3_vtab *pVtab){
   int rc = SQLITE_OK;
 
   assert( pTab->pWriteFd==0 );
+  if( pTab->zFile==0 ) return SQLITE_OK;
 
   /* Open a write fd on the file. Also load the entire central directory
   ** structure into memory. During the transaction any new file data is 

manifest

@@ -1,5 +1,5 @@
-C Refactor\snames.\s\sUse\s"small"\sinstead\sof\s"mini"\sto\sdescribe\sthe\ssmaller\nof\sthe\stwo\slookaside\smemory\sallocation\ssizes.
+C Merge\sthe\slatest\senhancements\sfrom\strunk.
-D 2019-12-31T15:12:34.090
+D 2020-01-02T21:41:59.033
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -94,7 +94,7 @@ F ext/fts3/fts3_snippet.c d62e40baa6a200586c3a425478109418f44926b1c318c870b59a42
 F ext/fts3/fts3_term.c f45a1e7c6ef464abb1231245d123dae12266b69e05cc56e14045b76591ae92d1
 F ext/fts3/fts3_test.c 73b16e229e517c1b1f0fb8e1046182a4e5dbc8dbe6eea8a5d4353fcce7dbbf39
 F ext/fts3/fts3_tokenize_vtab.c 1de9a61acfa2a0445ed989310c31839c57f6b6086dd9d5c97177ae734a17fd8b
-F ext/fts3/fts3_tokenizer.c a1ca0ab7c9c688ccb1d605645809f74f5df2bf4ca568e4bb37fb51113fd78e45
+F ext/fts3/fts3_tokenizer.c 4a36573c36fae7654de6fc56bd87f39d82100bb6db2cfd23d6e04d2a9c28d51d
 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d
@@ -182,7 +182,7 @@ F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b99
 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
 F ext/fts5/test/fts5hash.test a4cf51acad99bfc43c16fb74f9d22495dc221ae0701fc5e908ca963a9b26a02b
-F ext/fts5/test/fts5integrity.test c354fd693d9fb14018b33113aa2331265d915d51f67067a2cfa7873894c9ecf0
+F ext/fts5/test/fts5integrity.test 8ffabcd91b058d812aba3e3e0a06f76ce165ba402a18cca20e34204a7feec92e
 F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fdde7ea00e78ea283d227
 F ext/fts5/test/fts5lastrowid.test be98fe3e03235296585b72daad7aed5717ba0062bae5e5c18dd6e04e194c6b28
 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad
@@ -325,7 +325,7 @@ F ext/misc/vfsstat.c 77b5b4235c9f7f11eddf82487c0a422944ac2f132dafd5af3be7a68a057
 F ext/misc/vtablog.c 5538acd0c8ddaae372331bee11608d76973436b77d6a91e8635cfc9432fba5ae
 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd
 F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212
-F ext/misc/zipfile.c 185d060a595c79ebbec904e9d73b97b400dca94d2f910d351d6a8b5fcfe2c3d0
+F ext/misc/zipfile.c 442bd056690ceb82a125f2be53738c721357612b3ba5d9249220d4c1f85b2e3a
 F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64
 F ext/rbu/rbu.c 8681f6157db6adc82c34af24b14ea8a3be0146ad2a3b6c1d5da6cb8a5796c8ce
 F ext/rbu/rbu1.test 221d9c18a5e600ac9ac6b1810d99d9f99163a7909ba61597876ab6e4d4beb3d6
@@ -480,19 +480,19 @@ F src/date.c e1d8ac7102f3f283e63e13867acb0efa33861cf34f0faf4cdbaf9fa7a1eb7041
 F src/dbpage.c 135eb3b5e74f9ef74bde5cec2571192c90c86984fa534c88bf4a055076fa19b7
 F src/dbstat.c 6c407e549406c10fde9ac3987f6d734459205239ad370369bc5fcd683084a4fa
 F src/delete.c a5c59b9c0251cf7682bc52af0d64f09b1aefc6781a63592c8f1136f7b73c66e4
-F src/expr.c 8d4b6ac02f69e60b185cba0e38a76ba3f6f2122ee767e7de0f933fddec423e4a
+F src/expr.c bef2f4c18a90fec59575dba1e32cc7d120af4120105bf503b843507f03f1d378
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 92a248ec0fa4ed8ab60c98d9b188ce173aaf218f32e7737ba77deb2a684f9847
-F src/func.c ed33e38cd642058182a31a3f518f2e34f4bbe53aa483335705c153c4d3e50b12
+F src/func.c 259496e4856bd0a3215d16804992f3339f3e8db29f129a5a7285c341488bbe9c
-F src/global.c 89e36838a355f99cf86b118c3d4078fc3fc95cc9f992cbd70b422da8bfdf1789
+F src/global.c b1ae842e22899ba071f90d7bcfac4052ef8d81691bc7bd804835dee55d91175f
 F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19
 F src/hash.h 9d56a9079d523b648774c1784b74b89bd93fac7b365210157482e4319a468f38
 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
-F src/insert.c e43c6475b302d094ffa01d76b784c719001c9cf24c0b5af96b78e5b25dc54b83
+F src/insert.c 5ba8fd376f539240939ae76b5bc9fa7ad9a0d86e9914ecd11eb7002204138c11
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c d74f5e7bd51f3c9d283442473eb65aef359664efd6513591c03f01881c4ae2da
-F src/main.c e9e8da06e089933c3151951be34ac7212212dd8d1551f1d84a2f4416e7af56e0
+F src/main.c 07b5259e712bc479f0d7b51feaceab6ae3a4e08019a7ddad004562d56c317701
 F src/malloc.c eaa4dc9602ce28b077f7de2eb275db2be270c5cc56d7fec5466301bd9b80e2f5
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -526,19 +526,19 @@ F src/pragma.h ec3b31eac9b1df040f1cc8cb3d89bc06605c3b4cb3d76f833de8d6d6c3f77f04
 F src/prepare.c 6049beb71385f017af6fc320d2c75a4e50b75e280c54232442b785fbb83df057
 F src/printf.c 9be6945837c839ba57837b4bc3af349eba630920fa5532aa518816defe42a7d4
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c a76b7ac9848ae226f8dd718e607a830eb6ce9140d8c314735379ab560ad48df6
+F src/resolve.c 6cc3ce3d674b027b437e5b3c4f0c29be7995dc8abf27f1ee8b16e3d070e19d2f
 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
-F src/select.c bf8e7222602afe1fbce0c09e4300756b5ca5118f7224f414dd58debe6eb80b5e
+F src/select.c 8791103f21df458fdaba16c7d6452f92b00363f222f0f242661654f12455ea0a
-F src/shell.c.in 4a3a9e1c11847b1904f2b01d087af1c052f660902755abab457cab1756817ded
+F src/shell.c.in 90b002bf0054399cbbfac62dd752a9b05770427ba141bcba75eefbb0098f4280
 F src/sqlite.h.in 51f69c62ba3e980aca1e39badcaf9ad13f008774fe1bb8e7f57e3e456c656670
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 72af51aa4e912e14cd495fb6e7fac65f0940db80ed950d90911aff292cc47ce2
-F src/sqliteInt.h edc69ef8684086e8c195f969e47173ad960a0eb3f6298222b635e2ad3e0c1f98
+F src/sqliteInt.h 7f028954dc3602b5505064b1028ac671965bfd9905316767cb3298f945478ac0
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 9ff2210207c6c3b4d9631a8241a7d45ab1b26a0e9c84cb07a9b5ce2de9a3b278
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 8cd2600e8de23dff6cdf84d39f46ca57139b061b28f6f80b166bace17d52ab1c
-F src/test1.c c654981c1d86ebc90dd23fcc0969e6c85e28112f0acc2e2224a97a2a33e7c42f
+F src/test1.c 4d0ab2f67053a4fff87d1d3586ecc0e5322a1fc45dd4119ab11dc96de44f17a1
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644
 F src/test4.c 405834f6a93ec395cc4c9bb8ecebf7c3d8079e7ca16ae65e82d01afd229694bb
@@ -600,13 +600,13 @@ F src/upsert.c 2920de71b20f04fe25eb00b655d086f0ba60ea133c59d7fa3325c49838818e78
 F src/utf.c 2f0fac345c7660d5c5bd3df9e9d8d33d4c27f366bcfb09e07443064d751a0507
 F src/util.c 2c92bc706bbdb1c45a25180291e7e05a56e297aa5dd7b2bcd2b1c47e8bb05b17
 F src/vacuum.c 82dcec9e7b1afa980288718ad11bc499651c722d7b9f32933c4d694d91cb6ebf
-F src/vdbe.c 119ddfe16b35fbaaf5460c9b58a8eeb0cc5d43058342af36925411dfc73f9862
+F src/vdbe.c 459a233a60aba8b8b2462f42bd6e93623e13beafc5e0c40c6646e0527e444fd0
-F src/vdbe.h 3f068f00b23aebf392df142312ab5874588371c6d83e60d953f6d6b6453491c5
+F src/vdbe.h defd693289c7bb8d325f109be9490c77138061211a116827da7244b6015a4934
 F src/vdbeInt.h e02ccac0334f7c71c952210657e6e18de1917605887c7bc6167a80a17f62da18
 F src/vdbeapi.c 1252d80c548711e47a6d84dae88ed4e95d3fbb4e7bd0eaa1347299af7efddf02
-F src/vdbeaux.c 0a9716e47012ef018038c2e1dab9f701a6fb4429bb3ee1d4d0f49497519ace74
+F src/vdbeaux.c 3b2076c59217f964ff5fca348fa3f69c3157cbd5cafc549e9ea3252d8613c6a9
 F src/vdbeblob.c 253ed82894924c362a7fa3079551d3554cd1cdace39aa833da77d3bc67e7c1b1
-F src/vdbemem.c fa083086758379b52f8771d69424b273c7bd0f94413802404ee32cd5cc7cd870
+F src/vdbemem.c 05668cc1b44845736784f1ce9da46403dbf202c7c6d1a02205285cfc30e78f0d
 F src/vdbesort.c a3be032cc3fee0e3af31773af4a7a6f931b7230a34f53282ccf1d9a2a72343be
 F src/vdbetrace.c fa3bf238002f0bbbdfb66cc8afb0cea284ff9f148d6439bc1f6f2b4c3b7143f0
 F src/vtab.c a2fead3e97fca54fcf3f3db784e17c9ee2d39a0c5ad323e9d514855106300a86
@@ -614,11 +614,11 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 15a2845769f51ba132f9cf0b2c7a6887a91fc8437892dbcce9fcdc68b66d60a1
 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
 F src/walker.c a137468bf36c92e64d2275caa80c83902e3a0fc59273591b96c6416d3253d05d
-F src/where.c 602e5093556bbd9090c0a9bd834fec0717e3a4b0377a021d38091a6d554a1177
+F src/where.c 9353093c2a444580857006fc959494edc40e2393ac08a1f7e1eefe455c079cdb
-F src/whereInt.h a727b32260e12707a8c1bc29d7f7e9b6dc1a44551a45093d5968fbe570ff0c56
+F src/whereInt.h d2b771335083070ff82991cc43603e2db27b7ba1313da72de092c50c68f2be9c
-F src/wherecode.c a987d22b42e09b06f3a49596e0953b1cd28e568cc656681776edc0026cfac0cc
+F src/wherecode.c b8acf97f95de7398455e238036e96aeda8563d87c6b8108c967396431b6f2307
 F src/whereexpr.c 4b34be1434183e7bb8a05d4bf42bd53ea53021b0b060936fbd12062b4ff6b396
-F src/window.c 87795bb8293179cb8a92529264d49bd66b5bcad5e91cfc17dd8d3e66a2a77f88
+F src/window.c 659d613248f8bb8630f51409dc08235e4494c3c84162a535d9f88b38515f390a
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -626,17 +626,17 @@ F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 12106f0748e8e9bfc1a8e6840e203e051eae06a26ed13fc9fd5db108a8d6db54
 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87
 F test/all.test 2ecb8bbd52416642e41c9081182a8df05d42c75637afd4488aace78cc4b69e13
-F test/alter.test 16ed8d2470193f34bc711e51506ff1211ebfab8025ca3b9510ff2aef139874cb
+F test/alter.test 77f0092d137dd9470fc683b64ed92868e188462e713e52f48deae8902ea60b96
 F test/alter2.test a966ccfcddf9ce0a4e0e6ff1aca9e6e7948e0e242cd7e43fc091948521807687
 F test/alter3.test 9351a9f0c59ff9dddecccaaa2f777ffee5369870c63d30d3a74add815254ec0f
 F test/alter4.test 74b22251c5e9c48093cfc4921ed9c11b59df84634aeeb00e501773320beb8424
 F test/alterauth.test 63442ba61ceb0c1eeb63aac1f4f5cebfa509d352276059d27106ae256bafc959
 F test/alterauth2.test c0a1ddf5b93d93cb0d15ba7acaf0c5c6fb515bbe861ede75b2d3fabad33b6499
-F test/altercol.test 54374d2ba18af25bb24e23acf18a60270d4ec120b7ec0558078b59d5aa1a31ad
+F test/altercol.test 1d6a6fe698b81e626baea4881f5717f9bc53d7d07f1cd23ee7ad1b931f117ddf
 F test/alterlegacy.test 82022721ce0de29cedc9a7af63bc9fcc078b0ee000f8283b4b6ea9c3eab2f44b
 F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74a3fb80c9
 F test/altermalloc2.test fa7b1c1139ea39b8dec407cf1feb032ca8e0076bd429574969b619175ad0174b
-F test/altertab.test 4d8b79b0b88b62b90b710390df14fe99e0a3578345526886eaa550e28e3065dc
+F test/altertab.test bd61e5b73d495ec4707133db91b07f09d57e339d988de5ec5a76d34a2198e8f2
 F test/altertab2.test b0d62f323ca5dab42b0bc028c52e310ebdd13e655e8fac070fe622bad7852c2b
 F test/altertab3.test 155b8dc225ce484454a7fb4c8ba745680b6fa0fc3e08919cbbc19f9309d128ff
 F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f
@@ -727,7 +727,7 @@ F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4
 F test/capi3c.test 54e2dc0c8fd7c34ad1590d1be6864397da2438c95a9f5aee2f8fbc60c112e44b
 F test/capi3d.test aba917805573a03deed961a21f07a5a84505ad0a616f7e3fc1508844a15bccc4
 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe
-F test/cast.test 3619f0c58c2e4b2a94aa86e75607e497d34ef40ab74418e71aef7b4ca5155895
+F test/cast.test 2906ccab6a3ebd147ffa63304b635be903ce58264110d0a0eb4fd9939422bb53
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test b21a76546c2115af2674280566a8eba577e72adfec330c3d9a8a466d41f8eb0d
 F test/checkfault.test da6cb3d50247169efcb20bdf57863a3ccfa1d27d9e55cd324f0680096970f014
@@ -855,7 +855,7 @@ F test/exclusive.test d6ccc6acc5d660544f8e0cacaec2c620f8ebb42a764d783ab53430e260
 F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac
-F test/expr.test 7cb55e80aeb41d65fec968c08212505123063fea60bdc355d764d747670e9eea
+F test/expr.test 26cd01e8485bc48c8aa6a1add598e9ce1e706b4eb4f3f554e0b0223022e8c2cf
 F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8
 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9
 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79
@@ -867,7 +867,7 @@ F test/filter2.tcl 44e525497ce07382915f01bd29ffd0fa49dab3adb87253b5e5103ba8f9339
 F test/filter2.test 485cf95d1f6d6ceee5632201ca52a71868599836f430cdee42e5f7f14666e30a
 F test/filterfault.test c08fb491d698e8df6c122c98f7db1c65ffcfcad2c1ab0e07fa8a5be1b34eaa8b
 F test/fkey1.test d11dbb8a93ead9b5c46ae5d02da016d61245d47662fb2d844c99214f6163f768
-F test/fkey2.test d35d1c81e7569bdd2b872e91750f7098117d2e8291369f70b7e3d50a0e523dc2
+F test/fkey2.test 65c86b11127c11f80c0f450b3480321e0f087edea3031b9daa1978e3c020c91b
 F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49
 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d
 F test/fkey5.test 24dd28eb3d9f1b5a174f47e9899ace5facb08373a4223593c8c631e6cf9f7d5a
@@ -929,7 +929,7 @@ F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f
 F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
 F test/fts3ao.test 266989148fec6d9f1bb6c5382f7aa3dcea0e9cd444576e28dd2b9287ac7dd220
-F test/fts3atoken.test bef8a163490098a6b8a6ec5f5407269a3a15b9902c0fcf5e962825a81675b3a0
+F test/fts3atoken.test 2b2b0d7943eccf76e7a4887b557d4abcbb9279fbbb88e2bc2eba31c1817d11c7
 F test/fts3auto.test bfe0857bd0b69d68dd685a931b58486411a69f5794a7f6d6fe808bfa31a99614
 F test/fts3aux1.test 7a170e172afdbceb67f5baa05941fd4fbf56af42f61daa3d140f4b4bf4cb68f6
 F test/fts3aux2.test 2459e7fa3e22734aed237d1e2ae192f5541c4d8b218956ad2d90754977bf907f
@@ -1018,7 +1018,7 @@ F test/fuzzdata4.db b502c7d5498261715812dd8b3c2005bad08b3a26e6489414bd13926cd3e4
 F test/fuzzdata5.db e35f64af17ec48926481cfaf3b3855e436bd40d1cfe2d59a9474cb4b748a52a5
 F test/fuzzdata6.db 92a80e4afc172c24f662a10a612d188fb272de4a9bd19e017927c95f737de6d7
 F test/fuzzdata7.db e7a86fd83dda151d160445d542e32e5c6019c541b3a74c2a525b6ac640639711
-F test/fuzzdata8.db a0b7151bf505ee9735f6e5a3f7851ecb35121f1cd731ac677a77ee540f358939
+F test/fuzzdata8.db d702762f8bf21456a6b69d15dde3e3023733da01ed6f1c06fe50d0b2d161199f
 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8
 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14
 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536
@@ -1295,7 +1295,7 @@ F test/schema6.test e4bd1f23d368695eb9e7b51ef6e02ca0642ea2ab4a52579959826b5e7dce
 F test/schemafault.test 1936bceca55ac82c5efbcc9fc91a1933e45c8d1e1d106b9a7e56c972a5a2a51e
 F test/securedel.test 2f70b2449186a1921bd01ec9da407fbfa98c3a7a5521854c300c194b2ff09384
 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5
-F test/select1.test b887331202618dbdabed92446b661ebd95a55ef3923700af56ed71266e9c1157
+F test/select1.test 0aaefed5e770232effeba2d19b8dde01b639d617b844cb2492a75961e4119549
 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56
 F test/select3.test 3905450067c28766bc83ee397f6d87342de868baa60f2bcfd00f286dfbd62cb9
 F test/select4.test 5389d9895968d1196c457d59b3ee6515d771d328
@@ -1714,7 +1714,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2
 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d
 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc
-F test/window1.test cd6e2dafaa14ae26c995547013a9765356d910f473664497c926ea3f47510997
+F test/window1.test 30f38038f2878a28befa66c288099e5da131ba673799d73d0864f08375d503e1
 F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476
 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c
 F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03
@@ -1740,7 +1740,7 @@ F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f1982
 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64
 F test/without_rowid1.test 9cfb83705c506e3849fa7efc88a3c9a15f9a50bf9b1516b41757a7cef9bba8c3
 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99
-F test/without_rowid3.test ea4b59dd1b0d7f5f5e4b7cca978cdb905752a9d7c57dc4344a591dba765a3691
+F test/without_rowid3.test 392e6e12f275d11d931a8bc4580e573342f391639c87ffb631010a7b3cedfdc0
 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a
 F test/without_rowid5.test 89b1c587bd92a0590e440da33e7666bf4891572a
 F test/without_rowid6.test 8463b20098e9f75a501a9f17dfb42fffc79068eac0b2775fe56ef2281d2df45e
@@ -1749,7 +1749,7 @@ F test/wordcount.c d721a4b6fae93e6e33449700bce1686bc23257c27425bc3ef1599dc912ade
 F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc
 F test/zeroblob.test 07a5b11ab591d1f26c626945fb7f228f68b993533b2ada77273edf6ee29db174
 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc
-F test/zipfile.test aab99dc488586842bed415a1ce1a2327948064aa660c00243e3f14a1ff1cf282
+F test/zipfile.test 42603fac7f7d6fdb3e8690da9fd1492b89b2325b3946fcff683cc88437d84ce4
 F test/zipfile2.test 9903388a602a3834189857a985106ff95c3bba6a3969e0134127df991889db5d
 F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a168827edbbc2c
 F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb44
@@ -1853,7 +1853,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 39d55579376906f212271ce9b2d367e3ad029fb173f22c7253312b467970208a
+P 88d244983854cdc1a369c5df09ae00f1098784db768ba4e17b963d187dcb1009 7405e982257611803792617f5d8142c54f9831b4a030f4de8607888c525e6289
-R c6267a3dfb8286b9a7bf015a7e8fc93b
+R efdd5d9177183f4cebb9b751b044f27a
 U drh
-Z 457f22be9b5a945c48c466bed56b269c
+Z dc1b72a94a902754e9baf3ba10a5a0b3

manifest.uuid

@@ -1 +1 @@
-88d244983854cdc1a369c5df09ae00f1098784db768ba4e17b963d187dcb1009
+bd57e6d923d3b04f0a07aaf18bf389d2b2b7efc7c57e8cb37e6ef910662d8397

src/expr.c

@@ -3574,6 +3574,106 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){
   return iResult;
 }
 
+/*
+** Generate code to implement special SQL functions that are implemented
+** in-line rather than by using the usual callbacks.
+*/
+static int exprCodeInlineFunction(
+  Parse *pParse,        /* Parsing context */
+  ExprList *pFarg,      /* List of function arguments */
+  int iFuncId,          /* Function ID.  One of the INTFUNC_... values */
+  int target            /* Store function result in this register */
+){
+  int nFarg;
+  Vdbe *v = pParse->pVdbe;
+  assert( v!=0 );
+  assert( pFarg!=0 );
+  nFarg = pFarg->nExpr;
+  assert( nFarg>0 );  /* All in-line functions have at least one argument */
+  switch( iFuncId ){
+    case INLINEFUNC_coalesce: {
+      /* Attempt a direct implementation of the built-in COALESCE() and
+      ** IFNULL() functions.  This avoids unnecessary evaluation of
+      ** arguments past the first non-NULL argument.
+      */
+      int endCoalesce = sqlite3VdbeMakeLabel(pParse);
+      int i;
+      assert( nFarg>=2 );
+      sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target);
+      for(i=1; i<nFarg; i++){
+        sqlite3VdbeAddOp2(v, OP_NotNull, target, endCoalesce);
+        VdbeCoverage(v);
+        sqlite3ExprCode(pParse, pFarg->a[i].pExpr, target);
+      }
+      sqlite3VdbeResolveLabel(v, endCoalesce);
+      break;
+    }
+
+    default: {   
+      /* The UNLIKELY() function is a no-op.  The result is the value
+      ** of the first argument.
+      */
+      assert( nFarg==1 || nFarg==2 );
+      target = sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target);
+      break;
+    }
+
+  /***********************************************************************
+  ** Test-only SQL functions that are only usable if enabled
+  ** via SQLITE_TESTCTRL_INTERNAL_FUNCTIONS
+  */
+    case INLINEFUNC_expr_compare: {
+      /* Compare two expressions using sqlite3ExprCompare() */
+      assert( nFarg==2 );
+      sqlite3VdbeAddOp2(v, OP_Integer, 
+         sqlite3ExprCompare(0,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1),
+         target);
+      break;
+    }
+
+    case INLINEFUNC_expr_implies_expr: {
+      /* Compare two expressions using sqlite3ExprImpliesExpr() */
+      assert( nFarg==2 );
+      sqlite3VdbeAddOp2(v, OP_Integer, 
+         sqlite3ExprImpliesExpr(pParse,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1),
+         target);
+      break;
+    }
+
+    case INLINEFUNC_implies_nonnull_row: {
+      /* REsult of sqlite3ExprImpliesNonNullRow() */
+      Expr *pA1;
+      assert( nFarg==2 );
+      pA1 = pFarg->a[1].pExpr;
+      if( pA1->op==TK_COLUMN ){
+        sqlite3VdbeAddOp2(v, OP_Integer, 
+           sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable),
+           target);
+      }else{
+        sqlite3VdbeAddOp2(v, OP_Null, 0, target);
+      }
+      break;
+    }
+
+#ifdef SQLITE_DEBUG
+    case INLINEFUNC_affinity: {
+      /* The AFFINITY() function evaluates to a string that describes
+      ** the type affinity of the argument.  This is used for testing of
+      ** the SQLite type logic.
+      */
+      const char *azAff[] = { "blob", "text", "numeric", "integer", "real" };
+      char aff;
+      assert( nFarg==1 );
+      aff = sqlite3ExprAffinity(pFarg->a[0].pExpr);
+      sqlite3VdbeLoadString(v, target, 
+              (aff<=SQLITE_AFF_NONE) ? "none" : azAff[aff-SQLITE_AFF_BLOB]);
+      break;
+    }
+#endif
+  }
+  return target;
+}
+
 
 /*
 ** Generate code into the current Vdbe to evaluate the given
@@ -3954,47 +4054,10 @@ expr_code_doover:
         sqlite3ErrorMsg(pParse, "unknown function: %s()", zId);
         break;
       }
-
+      if( pDef->funcFlags & SQLITE_FUNC_INLINE ){
-      /* Attempt a direct implementation of the built-in COALESCE() and
+        return exprCodeInlineFunction(pParse, pFarg,
-      ** IFNULL() functions.  This avoids unnecessary evaluation of
+             SQLITE_PTR_TO_INT(pDef->pUserData), target);
-      ** arguments past the first non-NULL argument.
-      */
-      if( pDef->funcFlags & SQLITE_FUNC_COALESCE ){
-        int endCoalesce = sqlite3VdbeMakeLabel(pParse);
-        assert( nFarg>=2 );
-        sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target);
-        for(i=1; i<nFarg; i++){
-          sqlite3VdbeAddOp2(v, OP_NotNull, target, endCoalesce);
-          VdbeCoverage(v);
-          sqlite3ExprCode(pParse, pFarg->a[i].pExpr, target);
       }
-        sqlite3VdbeResolveLabel(v, endCoalesce);
-        break;
-      }
-
-      /* The UNLIKELY() function is a no-op.  The result is the value
-      ** of the first argument.
-      */
-      if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){
-        assert( nFarg>=1 );
-        return sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target);
-      }
-
-#ifdef SQLITE_DEBUG
-      /* The AFFINITY() function evaluates to a string that describes
-      ** the type affinity of the argument.  This is used for testing of
-      ** the SQLite type logic.
-      */
-      if( pDef->funcFlags & SQLITE_FUNC_AFFINITY ){
-        const char *azAff[] = { "blob", "text", "numeric", "integer", "real" };
-        char aff;
-        assert( nFarg==1 );
-        aff = sqlite3ExprAffinity(pFarg->a[0].pExpr);
-        sqlite3VdbeLoadString(v, target, 
-                (aff<=SQLITE_AFF_NONE) ? "none" : azAff[aff-SQLITE_AFF_BLOB]);
-        return target;
-      }
-#endif
 
       for(i=0; i<nFarg; i++){
         if( i<32 && sqlite3ExprIsConstant(pFarg->a[i].pExpr) ){
@@ -4078,7 +4141,7 @@ expr_code_doover:
         if( constMask==0 ){
           sqlite3ReleaseTempRange(pParse, r1, nFarg);
         }else{
-          sqlite3VdbeReleaseRegisters(pParse, r1, nFarg, constMask);
+          sqlite3VdbeReleaseRegisters(pParse, r1, nFarg, constMask, 1);
         }
       }
       return target;
@@ -4430,7 +4493,13 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){
   inReg = sqlite3ExprCodeTarget(pParse, pExpr, target);
   assert( pParse->pVdbe!=0 || pParse->db->mallocFailed );
   if( inReg!=target && pParse->pVdbe ){
-    sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target);
+    u8 op;
+    if( ExprHasProperty(pExpr,EP_Subquery) ){
+      op = OP_Copy;
+    }else{
+      op = OP_SCopy;
+    }
+    sqlite3VdbeAddOp2(pParse->pVdbe, op, inReg, target);
   }
 }
 
@@ -5297,12 +5366,13 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){
       return WRC_Prune;
 
     case TK_AND:
-      assert( pWalker->eCode==0 );
+      if( pWalker->eCode==0 ){
         sqlite3WalkExpr(pWalker, pExpr->pLeft);
         if( pWalker->eCode ){
           pWalker->eCode = 0;
           sqlite3WalkExpr(pWalker, pExpr->pRight);
         }
+      }
       return WRC_Prune;
 
     case TK_BETWEEN:
@@ -5730,7 +5800,7 @@ int sqlite3GetTempReg(Parse *pParse){
 */
 void sqlite3ReleaseTempReg(Parse *pParse, int iReg){
   if( iReg ){
-    sqlite3VdbeReleaseRegisters(pParse, iReg, 1, 0);
+    sqlite3VdbeReleaseRegisters(pParse, iReg, 1, 0, 0);
     if( pParse->nTempReg<ArraySize(pParse->aTempReg) ){
       pParse->aTempReg[pParse->nTempReg++] = iReg;
     }
@@ -5759,7 +5829,7 @@ void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){
     sqlite3ReleaseTempReg(pParse, iReg);
     return;
   }
-  sqlite3VdbeReleaseRegisters(pParse, iReg, nReg, 0);
+  sqlite3VdbeReleaseRegisters(pParse, iReg, nReg, 0, 0);
   if( nReg>pParse->nRangeReg ){
     pParse->nRangeReg = nReg;
     pParse->iRangeReg = iReg;

src/func.c

@@ -1907,12 +1907,20 @@ void sqlite3RegisterBuiltinFunctions(void){
   ** For peak efficiency, put the most frequently used function last.
   */
   static FuncDef aBuiltinFunc[] = {
+/***** Functions only available with SQLITE_TESTCTRL_INTERNAL_FUNCTIONS *****/
+    TEST_FUNC(implies_nonnull_row, 2, INLINEFUNC_implies_nonnull_row, 0),
+    TEST_FUNC(expr_compare,        2, INLINEFUNC_expr_compare,        0),
+    TEST_FUNC(expr_implies_expr,   2, INLINEFUNC_expr_implies_expr,   0),
+#ifdef SQLITE_DEBUG
+    TEST_FUNC(affinity,          1, INLINEFUNC_affinity, 0),
+#endif
+/***** Regular functions *****/
 #ifdef SQLITE_SOUNDEX
     FUNCTION(soundex,            1, 0, 0, soundexFunc      ),
 #endif
 #ifndef SQLITE_OMIT_LOAD_EXTENSION
-    VFUNCTION(load_extension,    1, 0, 0, loadExt          ),
+    SFUNCTION(load_extension,    1, 0, 0, loadExt          ),
-    VFUNCTION(load_extension,    2, 0, 0, loadExt          ),
+    SFUNCTION(load_extension,    2, 0, 0, loadExt          ),
 #endif
 #if SQLITE_USER_AUTHENTICATION
     FUNCTION(sqlite_crypt,       2, 0, 0, sqlite3CryptFunc ),
@@ -1921,12 +1929,9 @@ void sqlite3RegisterBuiltinFunctions(void){
     DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc  ),
     DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc  ),
 #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */
-    FUNCTION2(unlikely,          1, 0, 0, noopFunc,  SQLITE_FUNC_UNLIKELY),
+    INLINE_FUNC(unlikely,        1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY),
-    FUNCTION2(likelihood,        2, 0, 0, noopFunc,  SQLITE_FUNC_UNLIKELY),
+    INLINE_FUNC(likelihood,      2, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY),
-    FUNCTION2(likely,            1, 0, 0, noopFunc,  SQLITE_FUNC_UNLIKELY),
+    INLINE_FUNC(likely,          1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY),
-#ifdef SQLITE_DEBUG
-    FUNCTION2(affinity,          1, 0, 0, noopFunc,  SQLITE_FUNC_AFFINITY),
-#endif
 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
     FUNCTION2(sqlite_offset,     1, 0, 0, noopFunc,  SQLITE_FUNC_OFFSET|
                                                      SQLITE_FUNC_TYPEOF),
@@ -1959,7 +1964,7 @@ void sqlite3RegisterBuiltinFunctions(void){
     FUNCTION(upper,              1, 0, 0, upperFunc        ),
     FUNCTION(lower,              1, 0, 0, lowerFunc        ),
     FUNCTION(hex,                1, 0, 0, hexFunc          ),
-    FUNCTION2(ifnull,            2, 0, 0, noopFunc,  SQLITE_FUNC_COALESCE),
+    INLINE_FUNC(ifnull,          2, INLINEFUNC_coalesce, SQLITE_FUNC_COALESCE),
     VFUNCTION(random,            0, 0, 0, randomFunc       ),
     VFUNCTION(randomblob,        1, 0, 0, randomBlob       ),
     FUNCTION(nullif,             2, 0, 1, nullifFunc       ),
@@ -1999,7 +2004,7 @@ void sqlite3RegisterBuiltinFunctions(void){
 #endif
     FUNCTION(coalesce,           1, 0, 0, 0                ),
     FUNCTION(coalesce,           0, 0, 0, 0                ),
-    FUNCTION2(coalesce,         -1, 0, 0, noopFunc,  SQLITE_FUNC_COALESCE),
+    INLINE_FUNC(coalesce,       -1, INLINEFUNC_coalesce, SQLITE_FUNC_COALESCE),
   };
 #ifndef SQLITE_OMIT_ALTERTABLE
   sqlite3AlterFunctions();

src/global.c

@@ -267,7 +267,6 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
    0,                         /* xTestCallback */
 #endif
    0,                         /* bLocaltimeFault */
-   0,                         /* bInternalFunctions */
    0x7ffffffe,                /* iOnceResetThreshold */
    SQLITE_DEFAULT_SORTERREF_SIZE,   /* szSorterRef */
    0,                         /* iPrngSeed */

src/insert.c

@@ -1020,7 +1020,7 @@ void sqlite3Insert(
     **         goto C
     **      D: ...
     */
-    sqlite3VdbeReleaseRegisters(pParse, regData, pTab->nCol, 0);
+    sqlite3VdbeReleaseRegisters(pParse, regData, pTab->nCol, 0, 0);
     addrInsTop = addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm);
     VdbeCoverage(v);
     if( ipkColumn>=0 ){
@@ -1976,6 +1976,7 @@ void sqlite3GenerateConstraintChecks(
       sqlite3SetMakeRecordP5(v, pIdx->pTable);
     }
 #endif
+    sqlite3VdbeReleaseRegisters(pParse, regIdx, pIdx->nColumn, 0, 0);
 
     /* In an UPDATE operation, if this index is the PRIMARY KEY index 
     ** of a WITHOUT ROWID table and there has been no change the

src/main.c

@@ -3347,6 +3347,13 @@ static int openDatabase(
   }
 #endif
 
+#ifdef SQLITE_ENABLE_INTERNAL_FUNCTIONS
+  /* Testing use only!!! The -DSQLITE_ENABLE_INTERNAL_FUNCTIONS=1 compile-time
+  ** option gives access to internal functions by default.  
+  ** Testing use only!!! */
+  db->mDbFlags |= DBFLAG_InternalFunc;
+#endif
+
   /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking
   ** mode.  -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking
   ** mode.  Doing nothing at all also makes NORMAL the default.
@@ -4079,15 +4086,14 @@ int sqlite3_test_control(int op, ...){
       break;
     }
 
-    /*   sqlite3_test_control(SQLITE_TESTCTRL_INTERNAL_FUNCS, int onoff);
+    /*   sqlite3_test_control(SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, sqlite3*);
     **
-    ** If parameter onoff is non-zero, internal-use-only SQL functions
+    ** Toggle the ability to use internal functions on or off for
-    ** are visible to ordinary SQL.  This is useful for testing but is
+    ** the database connection given in the argument.
-    ** unsafe because invalid parameters to those internal-use-only functions
-    ** can result in crashes or segfaults.
     */
     case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: {
-      sqlite3GlobalConfig.bInternalFunctions = va_arg(ap, int);
+      sqlite3 *db = va_arg(ap, sqlite3*);
+      db->mDbFlags ^= DBFLAG_InternalFunc;
       break;
     }
 

src/resolve.c

@@ -880,7 +880,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
         }
         if( (pDef->funcFlags & SQLITE_FUNC_INTERNAL)!=0
          && pParse->nested==0
-         && sqlite3Config.bInternalFunctions==0
+         && (pParse->db->mDbFlags & DBFLAG_InternalFunc)==0
         ){
           /* Internal-use-only functions are disallowed unless the
           ** SQL is being compiled using sqlite3NestedParse() */

src/select.c

@@ -6177,7 +6177,7 @@ int sqlite3Select(
 #ifndef SQLITE_OMIT_WINDOWFUNC
     Window *pWin = p->pWin;      /* Master window object (or NULL) */
     if( pWin ){
-      sqlite3WindowCodeInit(pParse, pWin);
+      sqlite3WindowCodeInit(pParse, p);
     }
 #endif
     assert( WHERE_USE_LIMIT==SF_FixedLimit );

src/shell.c.in

@@ -9219,7 +9219,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       { "extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,"BOOLEAN"   },
     /*{ "fault_install",      SQLITE_TESTCTRL_FAULT_INSTALL, ""             },*/
       { "imposter",         SQLITE_TESTCTRL_IMPOSTER, "SCHEMA ON/OFF ROOTPAGE"},
-      { "internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, "BOOLEAN"   },
+      { "internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, "" },
       { "localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN"       },
       { "never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN"        },
       { "optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK"   },
@@ -9335,7 +9335,6 @@ static int do_meta_command(char *zLine, ShellState *p){
         /* sqlite3_test_control(int, int) */
         case SQLITE_TESTCTRL_ASSERT:
         case SQLITE_TESTCTRL_ALWAYS:
-        case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS:
           if( nArg==3 ){
             int opt = booleanValue(azArg[2]);
             rc2 = sqlite3_test_control(testctrl, opt);
@@ -9353,6 +9352,12 @@ static int do_meta_command(char *zLine, ShellState *p){
           }
           break;
 
+        /* sqlite3_test_control(sqlite3*) */
+        case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS:
+          rc2 = sqlite3_test_control(testctrl, p->db);
+          isOk = 3;
+          break;
+
         case SQLITE_TESTCTRL_IMPOSTER:
           if( nArg==5 ){
             rc2 = sqlite3_test_control(testctrl, p->db,

src/sqliteInt.h

@@ -1621,6 +1621,7 @@ struct sqlite3 {
 #define DBFLAG_Vacuum         0x0004  /* Currently in a VACUUM */
 #define DBFLAG_VacuumInto     0x0008  /* Currently running VACUUM INTO */
 #define DBFLAG_SchemaKnownOk  0x0010  /* Schema is known to be valid */
+#define DBFLAG_InternalFunc   0x0020  /* Allow use of internal functions */
 
 /*
 ** Bits of the sqlite3.dbOptFlags field that are used by the
@@ -1744,12 +1745,21 @@ struct FuncDestructor {
 #define SQLITE_FUNC_MINMAX   0x1000 /* True for min() and max() aggregates */
 #define SQLITE_FUNC_SLOCHNG  0x2000 /* "Slow Change". Value constant during a
                                     ** single query - might change over time */
-#define SQLITE_FUNC_AFFINITY 0x4000 /* Built-in affinity() function */
+#define SQLITE_FUNC_TEST     0x4000 /* Built-in testing functions */
 #define SQLITE_FUNC_OFFSET   0x8000 /* Built-in sqlite_offset() function */
 #define SQLITE_FUNC_WINDOW   0x00010000 /* Built-in window-only function */
 #define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */
 #define SQLITE_FUNC_DIRECT   0x00080000 /* Not for use in TRIGGERs or VIEWs */
 #define SQLITE_FUNC_SUBTYPE  0x00100000 /* Result likely to have sub-type */
+#define SQLITE_FUNC_INLINE   0x00200000 /* Functions implemented in-line */
+
+/* Identifier numbers for each in-line function */
+#define INLINEFUNC_coalesce             0
+#define INLINEFUNC_implies_nonnull_row  1
+#define INLINEFUNC_expr_implies_expr    2
+#define INLINEFUNC_expr_compare         3      
+#define INLINEFUNC_affinity             4
+#define INLINEFUNC_unlikely            99  /* Default case */
 
 /*
 ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
@@ -1765,6 +1775,22 @@ struct FuncDestructor {
 **   VFUNCTION(zName, nArg, iArg, bNC, xFunc)
 **     Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag.
 **
+**   SFUNCTION(zName, nArg, iArg, bNC, xFunc)
+**     Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and
+**     adds the SQLITE_DIRECTONLY flag.
+**
+**   INLINE_FUNC(zName, nArg, iFuncId, mFlags)
+**     zName is the name of a function that is implemented by in-line
+**     byte code rather than by the usual callbacks. The iFuncId
+**     parameter determines the function id.  The mFlags parameter is
+**     optional SQLITE_FUNC_ flags for this function.
+**
+**   TEST_FUNC(zName, nArg, iFuncId, mFlags)
+**     zName is the name of a test-only function implemented by in-line
+**     byte code rather than by the usual callbacks. The iFuncId
+**     parameter determines the function id.  The mFlags parameter is
+**     optional SQLITE_FUNC_ flags for this function.
+**
 **   DFUNCTION(zName, nArg, iArg, bNC, xFunc)
 **     Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and
 **     adds the SQLITE_FUNC_SLOCHNG flag.  Used for date & time functions
@@ -1804,6 +1830,16 @@ struct FuncDestructor {
 #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \
   {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
    SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} }
+#define SFUNCTION(zName, nArg, iArg, bNC, xFunc) \
+  {nArg, SQLITE_UTF8|SQLITE_DIRECTONLY, \
+   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} }
+#define INLINE_FUNC(zName, nArg, iArg, mFlags) \
+  {nArg, SQLITE_UTF8|SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \
+   SQLITE_INT_TO_PTR(iArg), 0, noopFunc, 0, 0, 0, #zName, {0} }
+#define TEST_FUNC(zName, nArg, iArg, mFlags) \
+  {nArg, SQLITE_UTF8|SQLITE_FUNC_INTERNAL|SQLITE_FUNC_TEST| \
+         SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \
+   SQLITE_INT_TO_PTR(iArg), 0, noopFunc, 0, 0, 0, #zName, {0} }
 #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \
   {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8, \
    0, 0, xFunc, 0, 0, 0, #zName, {0} }
@@ -3573,7 +3609,6 @@ struct Sqlite3Config {
   int (*xTestCallback)(int);        /* Invoked by sqlite3FaultSim() */
 #endif
   int bLocaltimeFault;              /* True to fail localtime() calls */
-  int bInternalFunctions;           /* Internal SQL functions are visible */
   int iOnceResetThreshold;          /* When to reset OP_Once counters */
   u32 szSorterRef;                  /* Min size in bytes to use sorter-refs */
   unsigned int iPrngSeed;           /* Alternative fixed seed for the PRNG */
@@ -3736,7 +3771,7 @@ Window *sqlite3WindowAlloc(Parse*, int, int, Expr*, int , Expr*, u8);
 void sqlite3WindowAttach(Parse*, Expr*, Window*);
 void sqlite3WindowLink(Select *pSel, Window *pWin);
 int sqlite3WindowCompare(Parse*, Window*, Window*, int);
-void sqlite3WindowCodeInit(Parse*, Window*);
+void sqlite3WindowCodeInit(Parse*, Select*);
 void sqlite3WindowCodeStep(Parse*, Select*, WhereInfo*, int, int);
 int sqlite3WindowRewrite(Parse*, Select*);
 int sqlite3ExpandSubquery(Parse*, struct SrcList_item*);

src/test1.c

@@ -6872,7 +6872,16 @@ static int SQLITE_TCLAPI test_test_control(
 
   iFlag = aVerb[iVerb].i;
   switch( iFlag ){
-    case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS:
+    case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: {
+      sqlite3 *db = 0;
+      if( objc!=3 ){
+        Tcl_WrongNumArgs(interp, 2, objv, "DB");
+        return TCL_ERROR;
+      }
+      if( getDbPointer(interp, Tcl_GetString(objv[2]), &db) ) return TCL_ERROR;
+      sqlite3_test_control(SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, db);
+      break;
+    }
     case SQLITE_TESTCTRL_LOCALTIME_FAULT: {
       int val;
       if( objc!=3 ){

src/vdbe.c

@@ -117,6 +117,26 @@ int sqlite3_found_count = 0;
 # define UPDATE_MAX_BLOBSIZE(P)
 #endif
 
+#ifdef SQLITE_DEBUG
+/* This routine provides a convenient place to set a breakpoint during
+** tracing with PRAGMA vdbe_trace=on.  The breakpoint fires right after
+** each opcode is printed.  Variables "pc" (program counter) and pOp are
+** available to add conditionals to the breakpoint.  GDB example:
+**
+**         break test_trace_breakpoint if pc=22
+**
+** Other useful labels for breakpoints include:
+**   test_addop_breakpoint(pc,pOp)
+**   sqlite3CorruptError(lineno)
+**   sqlite3MisuseError(lineno)
+**   sqlite3CantopenError(lineno)
+*/
+static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){
+  static int n = 0;
+  n++;
+}
+#endif
+
 /*
 ** Invoke the VDBE coverage callback, if that callback is defined.  This
 ** feature is used for test suite validation only and does not appear an
@@ -566,13 +586,28 @@ static void memTracePrint(Mem *p){
   if( p->flags & MEM_Subtype ) printf(" subtype=0x%02x", p->eSubtype);
 }
 static void registerTrace(int iReg, Mem *p){
-  printf("REG[%d] = ", iReg);
+  printf("R[%d] = ", iReg);
   memTracePrint(p);
+  if( p->pScopyFrom ){
+    printf(" <== R[%d]", (int)(p->pScopyFrom - &p[-iReg]));
+  }
   printf("\n");
   sqlite3VdbeCheckMemInvariants(p);
 }
 #endif
 
+#ifdef SQLITE_DEBUG
+/*
+** Show the values of all registers in the virtual machine.  Used for
+** interactive debugging.
+*/
+void sqlite3VdbeRegisterDump(Vdbe *v){
+  int i;
+  for(i=1; i<v->nMem; i++) registerTrace(i, v->aMem+i);
+}
+#endif /* SQLITE_DEBUG */
+
+
 #ifdef SQLITE_DEBUG
 #  define REGISTER_TRACE(R,M) if(db->flags&SQLITE_VdbeTrace)registerTrace(R,M)
 #else
@@ -738,6 +773,7 @@ int sqlite3VdbeExec(
 #ifdef SQLITE_DEBUG
     if( db->flags & SQLITE_VdbeTrace ){
       sqlite3VdbePrintOp(stdout, (int)(pOp - aOp), pOp);
+      test_trace_breakpoint((int)(pOp - aOp),pOp,p);
     }
 #endif
       
@@ -1335,8 +1371,13 @@ case OP_Move: {
     memAboutToChange(p, pOut);
     sqlite3VdbeMemMove(pOut, pIn1);
 #ifdef SQLITE_DEBUG
-    if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrom<pOut ){
+    pIn1->pScopyFrom = 0;
-      pOut->pScopyFrom += pOp->p2 - p1;
+    { int i;
+      for(i=1; i<p->nMem; i++){
+        if( aMem[i].pScopyFrom==pIn1 ){
+          aMem[i].pScopyFrom = pOut;
+        }
+      }
     }
 #endif
     Deephemeralize(pOut);
@@ -4954,7 +4995,11 @@ case OP_Delete: {
   sqlite3VdbeIncrWriteCounter(p, pC);
 
 #ifdef SQLITE_DEBUG
-  if( pOp->p4type==P4_TABLE && HasRowid(pOp->p4.pTab) && pOp->p5==0 ){
+  if( pOp->p4type==P4_TABLE
+   && HasRowid(pOp->p4.pTab)
+   && pOp->p5==0
+   && sqlite3BtreeCursorIsValidNN(pC->uc.pCursor)
+  ){
     /* If p5 is zero, the seek operation that positioned the cursor prior to
     ** OP_Delete will have also set the pC->movetoTarget field to the rowid of
     ** the row that is being deleted */
@@ -7770,7 +7815,7 @@ case OP_Abortable: {
 #endif
 
 #ifdef SQLITE_DEBUG
-/* Opcode:  ReleaseReg   P1 P2 P3 * *
+/* Opcode:  ReleaseReg   P1 P2 P3 * P5
 ** Synopsis: release r[P1@P2] mask P3
 **
 ** Release registers from service.  Any content that was in the
@@ -7785,10 +7830,12 @@ case OP_Abortable: {
 ** a change to the value of the source register for the OP_SCopy will no longer
 ** generate an assertion fault in sqlite3VdbeMemAboutToChange().
 **
-** TODO: Released registers ought to also have their datatype set to
+** If P5 is set, then all released registers have their type set
-** MEM_Undefined so that any subsequent attempt to read the released
+** to MEM_Undefined so that any subsequent attempt to read the released
 ** register (before it is reinitialized) will generate an assertion fault.
-** However, there are places in the code generator which release registers
+**
+** P5 ought to be set on every call to this opcode.
+** However, there are places in the code generator will release registers
 ** before their are used, under the (valid) assumption that the registers
 ** will not be reallocated for some other purpose before they are used and
 ** hence are safe to release.
@@ -7809,7 +7856,7 @@ case OP_ReleaseReg: {
   for(i=0; i<pOp->p2; i++, pMem++){
     if( i>=32 || (constMask & MASKBIT32(i))==0 ){
       pMem->pScopyFrom = 0;
-      /* MemSetTypeFlag(pMem, MEM_Undefined); // See the TODO */
+      if( i<32 && pOp->p5 ) MemSetTypeFlag(pMem, MEM_Undefined);
     }
   }
   break;

src/vdbe.h

@@ -233,9 +233,9 @@ void sqlite3VdbeJumpHere(Vdbe*, int addr);
 int sqlite3VdbeChangeToNoop(Vdbe*, int addr);
 int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op);
 #ifdef SQLITE_DEBUG
-  void sqlite3VdbeReleaseRegisters(Parse*,int addr, int n, u32 mask);
+  void sqlite3VdbeReleaseRegisters(Parse*,int addr, int n, u32 mask, int);
 #else
-# define sqlite3VdbeReleaseRegisters(P,A,N,M)
+# define sqlite3VdbeReleaseRegisters(P,A,N,M,F)
 #endif
 void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N);
 void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type);

src/vdbeaux.c

@@ -194,9 +194,16 @@ static int growOpArray(Vdbe *v, int nOp){
 #ifdef SQLITE_DEBUG
 /* This routine is just a convenient place to set a breakpoint that will
 ** fire after each opcode is inserted and displayed using
-** "PRAGMA vdbe_addoptrace=on".
+** "PRAGMA vdbe_addoptrace=on".  Parameters "pc" (program counter) and
+** pOp are available to make the breakpoint conditional.
+**
+** Other useful labels for breakpoints include:
+**   test_trace_breakpoint(pc,pOp)
+**   sqlite3CorruptError(lineno)
+**   sqlite3MisuseError(lineno)
+**   sqlite3CantopenError(lineno)
 */
-static void test_addop_breakpoint(void){
+static void test_addop_breakpoint(int pc, Op *pOp){
   static int n = 0;
   n++;
 }
@@ -249,7 +256,7 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){
 #ifdef SQLITE_DEBUG
   if( p->db->flags & SQLITE_VdbeAddopTrace ){
     sqlite3VdbePrintOp(0, i, &p->aOp[i]);
-    test_addop_breakpoint();
+    test_addop_breakpoint(i, &p->aOp[i]);
   }
 #endif
 #ifdef VDBE_PROFILE
@@ -1191,8 +1198,17 @@ int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){
 ** Generate an OP_ReleaseReg opcode to indicate that a range of
 ** registers, except any identified by mask, are no longer in use.
 */
-void sqlite3VdbeReleaseRegisters(Parse *pParse, int iFirst, int N, u32 mask){
+void sqlite3VdbeReleaseRegisters(
+  Parse *pParse,       /* Parsing context */
+  int iFirst,          /* Index of first register to be released */
+  int N,               /* Number of registers to release */
+  u32 mask,            /* Mask of registers to NOT release */
+  int bUndefine        /* If true, mark registers as undefined */
+){
+  if( N==0 ) return;
   assert( pParse->pVdbe );
+  assert( iFirst>=1 );
+  assert( iFirst+N-1<=pParse->nMem );
   while( N>0 && (mask&1)!=0 ){
     mask >>= 1;
     iFirst++;
@@ -1204,6 +1220,7 @@ void sqlite3VdbeReleaseRegisters(Parse *pParse, int iFirst, int N, u32 mask){
   }
   if( N>0 ){
     sqlite3VdbeAddOp3(pParse->pVdbe, OP_ReleaseReg, iFirst, N, *(int*)&mask);
+    if( bUndefine ) sqlite3VdbeChangeP5(pParse->pVdbe, 1);
   }
 }
 #endif /* SQLITE_DEBUG */

src/vdbemem.c

@@ -948,14 +948,18 @@ int sqlite3VdbeMemTooBig(Mem *p){
 ** its link to a shallow copy and by marking any current shallow
 ** copies of this cell as invalid.
 **
-** This is used for testing and debugging only - to make sure shallow
+** This is used for testing and debugging only - to help ensure that shallow
-** copies are not misused.
+** copies (created by OP_SCopy) are not misused.
 */
 void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){
   int i;
   Mem *pX;
-  for(i=0, pX=pVdbe->aMem; i<pVdbe->nMem; i++, pX++){
+  for(i=1, pX=pVdbe->aMem+1; i<pVdbe->nMem; i++, pX++){
     if( pX->pScopyFrom==pMem ){
+      if( pVdbe->db->flags & SQLITE_VdbeTrace ){
+        sqlite3DebugPrintf("Invalidate R[%d] due to change in R[%d]\n",
+          (int)(pX - pVdbe->aMem), (int)(pMem - pVdbe->aMem));
+      }
       /* If pX is marked as a shallow copy of pMem, then verify that
       ** no significant changes have been made to pX since the OP_SCopy.
       ** A significant change would indicated a missed call to this
@@ -964,7 +968,9 @@ void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){
       ** same. */
       u16 mFlags = pMem->flags & pX->flags & pX->mScopyFlags;
       assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i );
-      assert( (mFlags&MEM_Real)==0 || pMem->u.r==pX->u.r );
+      /* assert( (mFlags&MEM_Real)==0 || pMem->u.r==pX->u.r ); */
+      /*                                          ^^           */
+      /*       Cannot reliably compare doubles for equality    */
       assert( (mFlags&MEM_Str)==0  || (pMem->n==pX->n && pMem->z==pX->z) );
       assert( (mFlags&MEM_Blob)==0  || sqlite3BlobCompare(pMem,pX)==0 );
       
@@ -978,7 +984,6 @@ void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){
 }
 #endif /* SQLITE_DEBUG */
 
-
 /*
 ** Make an shallow copy of pFrom into pTo.  Prior contents of
 ** pTo are freed.  The pFrom->z field is not duplicated.  If
@@ -1124,10 +1129,19 @@ int sqlite3VdbeMemSetStr(
 
   pMem->n = nByte;
   pMem->flags = flags;
-  pMem->enc = (enc==0 ? SQLITE_UTF8 : enc);
+  if( enc ){
+    pMem->enc = enc;
+#ifdef SQLITE_ENABLE_SESSION
+  }else if( pMem->db==0 ){
+    pMem->enc = SQLITE_UTF8;
+#endif
+  }else{
+    assert( pMem->db!=0 );
+    pMem->enc = ENC(pMem->db);
+  }
 
 #ifndef SQLITE_OMIT_UTF16
-  if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){
+  if( enc>SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){
     return SQLITE_NOMEM_BKPT;
   }
 #endif

src/where.c

@@ -1942,6 +1942,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
     pWInfo->pLoops = p->pNextLoop;
     whereLoopDelete(db, p);
   }
+  assert( pWInfo->pExprMods==0 );
   sqlite3DbFreeNN(db, pWInfo);
 }
 
@@ -5440,6 +5441,14 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     }
   }
 
+  /* Undo all Expr node modifications */
+  while( pWInfo->pExprMods ){
+    WhereExprMod *p = pWInfo->pExprMods;
+    pWInfo->pExprMods = p->pNext;
+    memcpy(p->pExpr, &p->orig, sizeof(p->orig));
+    sqlite3DbFree(db, p);
+  }
+
   /* Final cleanup
   */
   pParse->nQueryLoop = pWInfo->savedNQueryLoop;

src/whereInt.h

@@ -433,6 +433,20 @@ struct WhereLoopBuilder {
 # define SQLITE_QUERY_PLANNER_LIMIT_INCR 1000
 #endif
 
+/*
+** Each instance of this object records a change to a single node
+** in an expression tree to cause that node to point to a column
+** of an index rather than an expression or a virtual column.  All
+** such transformations need to be undone at the end of WHERE clause
+** processing.
+*/
+typedef struct WhereExprMod WhereExprMod;
+struct WhereExprMod {
+  WhereExprMod *pNext;  /* Next translation on a list of them all */
+  Expr *pExpr;          /* The Expr node that was transformed */
+  Expr orig;            /* Original value of the Expr node */
+};
+
 /*
 ** The WHERE clause processing routine has two halves.  The
 ** first part does the start of the WHERE loop and the second
@@ -449,24 +463,25 @@ struct WhereInfo {
   ExprList *pOrderBy;       /* The ORDER BY clause or NULL */
   ExprList *pResultSet;     /* Result set of the query */
   Expr *pWhere;             /* The complete WHERE clause */
-  LogEst iLimit;            /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */
   int aiCurOnePass[2];      /* OP_OpenWrite cursors for the ONEPASS opt */
   int iContinue;            /* Jump here to continue with next record */
   int iBreak;               /* Jump here to break out of the loop */
   int savedNQueryLoop;      /* pParse->nQueryLoop outside the WHERE loop */
   u16 wctrlFlags;           /* Flags originally passed to sqlite3WhereBegin() */
+  LogEst iLimit;            /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */
   u8 nLevel;                /* Number of nested loop */
   i8 nOBSat;                /* Number of ORDER BY terms satisfied by indices */
-  u8 sorted;                /* True if really sorted (not just grouped) */
   u8 eOnePass;              /* ONEPASS_OFF, or _SINGLE, or _MULTI */
-  u8 bDeferredSeek;         /* Uses OP_DeferredSeek */
-  u8 untestedTerms;         /* Not all WHERE terms resolved by outer loop */
   u8 eDistinct;             /* One of the WHERE_DISTINCT_* values */
-  u8 bOrderedInnerLoop;     /* True if only the inner-most loop is ordered */
+  u8 bDeferredSeek : 1;     /* Uses OP_DeferredSeek */
+  u8 untestedTerms : 1;     /* Not all WHERE terms resolved by outer loop */
+  u8 bOrderedInnerLoop : 1; /* True if only the inner-most loop is ordered */
+  u8 sorted : 1;            /* True if really sorted (not just grouped) */
+  LogEst nRowOut;           /* Estimated number of output rows */
   int iTop;                 /* The very beginning of the WHERE loop */
   WhereLoop *pLoops;        /* List of all WhereLoop objects */
+  WhereExprMod *pExprMods;  /* Expression modifications */
   Bitmask revMask;          /* Mask of ORDER BY terms that need reversing */
-  LogEst nRowOut;           /* Estimated number of output rows */
   WhereClause sWC;          /* Decomposition of the WHERE clause */
   WhereMaskSet sMaskSet;    /* Map cursor numbers to bitmasks */
   WhereLevel a[1];          /* Information about each nest loop in WHERE */

src/wherecode.c

@@ -1113,8 +1113,23 @@ typedef struct IdxExprTrans {
   int iIdxCur;       /* The cursor for the index */
   int iIdxCol;       /* The column for the index */
   int iTabCol;       /* The column for the table */
+  WhereInfo *pWInfo; /* Complete WHERE clause information */
+  sqlite3 *db;       /* Database connection (for malloc()) */
 } IdxExprTrans;
 
+/*
+** Preserve pExpr on the WhereETrans list of the WhereInfo.
+*/
+static void preserveExpr(IdxExprTrans *pTrans, Expr *pExpr){
+  WhereExprMod *pNew;
+  pNew = sqlite3DbMallocRaw(pTrans->db, sizeof(*pNew));
+  if( pNew==0 ) return;
+  pNew->pNext = pTrans->pWInfo->pExprMods;
+  pTrans->pWInfo->pExprMods = pNew;
+  pNew->pExpr = pExpr;
+  memcpy(&pNew->orig, pExpr, sizeof(*pExpr));
+}
+
 /* The walker node callback used to transform matching expressions into
 ** a reference to an index column for an index on an expression.
 **
@@ -1124,6 +1139,7 @@ typedef struct IdxExprTrans {
 static int whereIndexExprTransNode(Walker *p, Expr *pExpr){
   IdxExprTrans *pX = p->u.pIdxTrans;
   if( sqlite3ExprCompare(0, pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){
+    preserveExpr(pX, pExpr);
     pExpr->affExpr = sqlite3ExprAffinity(pExpr);
     pExpr->op = TK_COLUMN;
     pExpr->iTable = pX->iIdxCur;
@@ -1147,6 +1163,7 @@ static int whereIndexExprTransColumn(Walker *p, Expr *pExpr){
     IdxExprTrans *pX = p->u.pIdxTrans;
     if( pExpr->iTable==pX->iTabCur && pExpr->iColumn==pX->iTabCol ){
       assert( pExpr->y.pTab!=0 );
+      preserveExpr(pX, pExpr);
       pExpr->affExpr = sqlite3TableColumnAffinity(pExpr->y.pTab,pExpr->iColumn);
       pExpr->iTable = pX->iIdxCur;
       pExpr->iColumn = pX->iIdxCol;
@@ -1188,6 +1205,8 @@ static void whereIndexExprTrans(
   w.u.pIdxTrans = &x;
   x.iTabCur = iTabCur;
   x.iIdxCur = iIdxCur;
+  x.pWInfo = pWInfo;
+  x.db = pWInfo->pParse->db;
   for(iIdxCol=0; iIdxCol<pIdx->nColumn; iIdxCol++){
     i16 iRef = pIdx->aiColumn[iIdxCol];
     if( iRef==XN_EXPR ){

src/window.c

@@ -1033,10 +1033,6 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){
         p->pSrc->a[0].pTab = pTab;
         pTab = pTab2;
       }
-      sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, pSublist->nExpr);
-      sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+1, pMWin->iEphCsr);
-      sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+2, pMWin->iEphCsr);
-      sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+3, pMWin->iEphCsr);
     }else{
       sqlite3SelectDelete(db, pSub);
     }
@@ -1308,10 +1304,17 @@ int sqlite3WindowCompare(Parse *pParse, Window *p1, Window *p2, int bFilter){
 ** to begin iterating through the sub-query results. It is used to allocate
 ** and initialize registers and cursors used by sqlite3WindowCodeStep().
 */
-void sqlite3WindowCodeInit(Parse *pParse, Window *pMWin){
+void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){
+  int nEphExpr = pSelect->pSrc->a[0].pSelect->pEList->nExpr;
+  Window *pMWin = pSelect->pWin;
   Window *pWin;
   Vdbe *v = sqlite3GetVdbe(pParse);
 
+  sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, nEphExpr);
+  sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+1, pMWin->iEphCsr);
+  sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+2, pMWin->iEphCsr);
+  sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+3, pMWin->iEphCsr);
+
   /* Allocate registers to use for PARTITION BY values, if any. Initialize
   ** said registers to NULL.  */
   if( pMWin->pPartition ){

test/alter.test

@@ -684,7 +684,7 @@ do_test alter-8.2 {
 # alter-9.X - Special test: Make sure the sqlite_rename_column() and
 # rename_table() functions do not crash when handed bad input.
 #
-sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 1
+sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
 do_test alter-9.1 {
   execsql {SELECT SQLITE_RENAME_COLUMN(0,0,0,0,0,0,0,0,0)}
 } {{}}
@@ -697,7 +697,7 @@ foreach {tn sql} {
     catch { execsql $sql }
   } 1
 }
-sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 0
+sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
 
 # If the INTERNAL_FUNCTIONS test-control is disabled (which is the default),
 # then the sqlite_rename_table() SQL function is not accessible to ordinary SQL.

test/altercol.test

@@ -618,7 +618,7 @@ foreach {tn trigger error} {
 #-------------------------------------------------------------------------
 # Passing invalid parameters directly to sqlite_rename_column().
 #
-sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 1
+sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
 do_execsql_test 14.1 {
   CREATE TABLE ddd(sql, type, object, db, tbl, icol, znew, bquote);
   INSERT INTO ddd VALUES(
@@ -641,7 +641,7 @@ do_execsql_test 14.2 {
   sqlite_rename_column(sql, type, object, db, tbl, icol, znew, bquote, 0)
   FROM ddd;
 } {{} {} {} {}}
-sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 0
+sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
 
 # If the INTERNAL_FUNCTIONS test-control is disabled (which is the default)
 # then the sqlite_rename_table() SQL function is not accessible to

test/altertab.test

@@ -240,13 +240,13 @@ ifcapable vtab {
     );
   } {}
   
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 1
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   do_execsql_test 7.2 {
     SELECT 
     sqlite_rename_table(db, 0, 0, sql, zOld, zNew, bTemp)
     FROM ddd;
   } {{} {} {}}
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 0
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
 }
 
 #-------------------------------------------------------------------------

test/cast.test

@@ -461,6 +461,18 @@ do_execsql_test cast-7.43 {
   SELECT CAST('-1.0' AS numeric);
 } -1
 
+ifcapable utf16 {
+  reset_db
+  execsql { PRAGMA encoding='utf16' }
+
+  do_execsql_test cast-8.1 {
+    SELECT quote(X'310032003300')==quote(substr(X'310032003300', 1))
+  } 1
+  do_execsql_test cast-8.2 {
+    SELECT CAST(X'310032003300' AS TEXT)
+         ==CAST(substr(X'310032003300', 1) AS TEXT)
+  } 1
+}
 
 
 finish_test

test/expr.test

@@ -1036,4 +1036,24 @@ foreach {tn val} [list 1 NaN 2 -NaN 3 NaN0 4 -NaN0 5 Inf 6 -Inf] {
   } {0}
 }
 
+reset_db
+sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
+do_execsql_test expr-16.1 {
+  CREATE TABLE t1(a,b,c);
+  CREATE TABLE dual(dummy);
+  INSERT INTO dual VALUES('X');
+} {}
+do_execsql_test expr-16.100 {
+  SELECT implies_nonnull_row( (b=1 AND 0)>(b=3 AND 0),a)
+  FROM dual LEFT JOIN t1;
+} {0}
+do_execsql_test expr-16.101 {
+  SELECT implies_nonnull_row( (b=1 AND 0)>(b=3 AND a=4),a)
+  FROM dual LEFT JOIN t1;
+} {1}
+do_execsql_test expr-16.102 {
+  SELECT implies_nonnull_row( (b=1 AND a=2)>(b=3 AND a=4),a)
+  FROM dual LEFT JOIN t1;
+} {1}
+
 finish_test

test/fkey2.test

@@ -987,7 +987,7 @@ ifcapable altertable {
         'main', 'table', 't1', $zCreate, $zOld, $zNew, 0
     )}
   }
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 1
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   do_test fkey2-14.2.1.1 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES t2)} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
@@ -997,7 +997,7 @@ ifcapable altertable {
   do_test fkey2-14.2.1.3 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES "t2")} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 0
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   
   # Test ALTER TABLE RENAME TABLE a bit.
   #
@@ -1070,7 +1070,7 @@ ifcapable altertable {
     }
   } {{CREATE TABLE t2(a, b, c REFERENCES t1, d DEFAULT NULL REFERENCES t1, e REFERENCES t1 DEFAULT NULL, h DEFAULT 'text' REFERENCES t1)}}
 
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 1
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   do_test fkey2-14.2tmp.1.1 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES t2)} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
@@ -1080,7 +1080,7 @@ ifcapable altertable {
   do_test fkey2-14.2tmp.1.3 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES "t2")} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 0
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   
   # Test ALTER TABLE RENAME TABLE a bit.
   #
@@ -1154,7 +1154,7 @@ ifcapable altertable {
     }
   } {{CREATE TABLE t2(a, b, c REFERENCES t1, d DEFAULT NULL REFERENCES t1, e REFERENCES t1 DEFAULT NULL, h DEFAULT 'text' REFERENCES t1)}}
 
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 1
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   do_test fkey2-14.2aux.1.1 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES t2)} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
@@ -1164,7 +1164,7 @@ ifcapable altertable {
   do_test fkey2-14.2aux.1.3 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES "t2")} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 0
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   
   # Test ALTER TABLE RENAME TABLE a bit.
   #

test/fts3atoken.test

@@ -129,6 +129,29 @@ do_test fts3atoken-1.9 {
   }
 } {1 blob blob blob blob}
 
+# 2019-12-31:  The fts3_tokenizer() function can never be invoked from
+# within a trigger or view.
+#
+do_catchsql_test fts3atoken-1.10 {
+  CREATE VIEW v110(x) AS
+      SELECT fts3_tokenizer('tok110', fts3_tokenizer('simple')) IS NULL;
+} {0 {}}
+do_catchsql_test fts3atoken-1.11 {
+  SELECT * FROM v110;
+} {1 {fts3_tokenizer() prohibited in triggers and views}}
+do_catchsql_test fts3atoken-1.12 {
+  CREATE TABLE t110(a,b);
+  CREATE TRIGGER r110 AFTER INSERT ON t110 BEGIN
+      SELECT fts3_tokenizer('tok110', fts3_tokenizer('simple')) IS NULL;
+  END;
+} {0 {}}
+do_catchsql_test fts3atoken-1.13 {
+  INSERT INTO t110(a,b) VALUES(1,2);
+} {1 {fts3_tokenizer() prohibited in triggers and views}}
+do_catchsql_test fts3atoken-1.14 {
+  SELECT * FROM t110;
+} {0 {}}
+
 #--------------------------------------------------------------------------
 # Test cases fts3atoken-2.* test error cases in the scalar function based
 # API for getting and setting tokenizers.

test/select1.test

@@ -1184,5 +1184,18 @@ do_catchsql_test select-19.21 {
     ORDER BY 1;
 } {1 {table t1 has 1 columns but 15 values were supplied}}
 
+# 2020-01-01 Found by Yongheng's fuzzer
+#
+reset_db
+do_execsql_test select-20.10 {
+  CREATE TABLE t1 (
+    a INTEGER PRIMARY KEY,
+    b AS('Y') UNIQUE
+  );
+  INSERT INTO t1(a) VALUES (10);
+  SELECT * FROM t1 JOIN t1 USING(a,b)
+   WHERE ((SELECT t1.a FROM t1 AS x GROUP BY b) AND b=0)
+      OR a = 10;
+} {10 Y}
 
 finish_test

test/window1.test

@@ -1531,4 +1531,27 @@ do_execsql_test 45.2 {
   );
 } {2000 2000 10000}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 46.1 {
+  CREATE TABLE t1 (a);
+  CREATE INDEX i1 ON t1(a);
+
+  INSERT INTO t1 VALUES (10);
+}
+
+do_execsql_test 46.2 {
+  SELECT (SELECT sum(a) OVER(ORDER BY a)) FROM t1
+} 10
+
+do_execsql_test 46.3 {
+  SELECT * FROM t1 WHERE (SELECT sum(a) OVER(ORDER BY a));
+} 10
+
+do_execsql_test 46.4 {
+  SELECT * FROM t1 NATURAL JOIN t1
+    WHERE a=1
+    OR ((SELECT sum(a)OVER(ORDER BY a)) AND a<=10)
+} 10
+
 finish_test

test/without_rowid3.test

@@ -953,7 +953,7 @@ ifcapable altertable {
         'main', 'table', 't1', $zCreate, $zOld, $zNew, 0
     )}
   }
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 1
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   do_test without_rowid3-14.2.1.1 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES t2)} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
@@ -963,7 +963,7 @@ ifcapable altertable {
   do_test without_rowid3-14.2.1.3 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES "t2")} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 0
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   
   # Test ALTER TABLE RENAME TABLE a bit.
   #
@@ -1039,7 +1039,7 @@ ifcapable altertable {
     }
   } {{CREATE TABLE t2(a, b, c REFERENCES t1, d DEFAULT NULL REFERENCES t1, e REFERENCES t1 DEFAULT NULL, h DEFAULT 'text' REFERENCES t1)}}
 
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 1
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   do_test without_rowid3-14.2tmp.1.1 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES t2)} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
@@ -1049,7 +1049,7 @@ ifcapable altertable {
   do_test without_rowid3-14.2tmp.1.3 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES "t2")} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 0
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   
   # Test ALTER TABLE RENAME TABLE a bit.
   #
@@ -1126,7 +1126,7 @@ ifcapable altertable {
     }
   } {{CREATE TABLE t2(a, b, c REFERENCES t1, d DEFAULT NULL REFERENCES t1, e REFERENCES t1 DEFAULT NULL, h DEFAULT 'text' REFERENCES t1)}}
 
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 1
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   do_test without_rowid3-14.2aux.1.1 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES t2)} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
@@ -1136,7 +1136,7 @@ ifcapable altertable {
   do_test without_rowid3-14.2aux.1.3 {
     test_rename_parent {CREATE TABLE t1(a REFERENCES "t2")} t2 t3
   } {{CREATE TABLE t1(a REFERENCES "t3")}}
-  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 0
+  sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
   
   # Test ALTER TABLE RENAME TABLE a bit.
   #

test/zipfile.test

@@ -834,6 +834,11 @@ do_execsql_test 15.20 {
   REPLACE INTO t2 values(null,null,null,null,null,10,null);
 } {}
 
+# 2020-01-02 Yongheng fuzzer discovery
+#
+do_catchsql_test 16.10 {
+  DELETE FROM zipfile;
+} {1 {zipfile() function requires an argument}}
 
 
 finish_test