diff --git a/Makefile.in b/Makefile.in index 646cb39756..4929ce313c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -359,6 +359,7 @@ TESTSRC = \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ + $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_demovfs.c \ diff --git a/Makefile.msc b/Makefile.msc index 4aaa2894a7..64175a0679 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -828,6 +828,7 @@ TESTSRC = \ $(TOP)\src\test_autoext.c \ $(TOP)\src\test_async.c \ $(TOP)\src\test_backup.c \ + $(TOP)\src\test_blob.c \ $(TOP)\src\test_btree.c \ $(TOP)\src\test_config.c \ $(TOP)\src\test_demovfs.c \ diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index db175acdbb..8fb4c25daa 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -183,7 +183,7 @@ static int isVowel(const char *z){ ** by a consonant. ** ** In this routine z[] is in reverse order. So we are really looking -** for an instance of of a consonant followed by a vowel. +** for an instance of a consonant followed by a vowel. */ static int m_gt_0(const char *z){ while( isVowel(z) ){ z++; } diff --git a/ext/misc/eval.c b/ext/misc/eval.c index a5e297ad38..71b6b69f20 100644 --- a/ext/misc/eval.c +++ b/ext/misc/eval.c @@ -21,11 +21,11 @@ SQLITE_EXTENSION_INIT1 ** Structure used to accumulate the output */ struct EvalResult { - char *z; /* Accumulated output */ - const char *zSep; /* Separator */ - int szSep; /* Size of the separator string */ - int nAlloc; /* Number of bytes allocated for z[] */ - int nUsed; /* Number of bytes of z[] actually used */ + char *z; /* Accumulated output */ + const char *zSep; /* Separator */ + int szSep; /* Size of the separator string */ + sqlite3_int64 nAlloc; /* Number of bytes allocated for z[] */ + sqlite3_int64 nUsed; /* Number of bytes of z[] actually used */ }; /* @@ -37,10 +37,13 @@ static int callback(void *pCtx, int argc, char **argv, char **colnames){ for(i=0; inUsed+p->szSep+1 > p->nAlloc ){ + if( (sqlite3_int64)sz+p->nUsed+p->szSep+1 > p->nAlloc ){ char *zNew; p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1; - zNew = sqlite3_realloc(p->z, p->nAlloc); + /* Using sqlite3_realloc64() would be better, but it is a recent + ** addition and will cause a segfault if loaded by an older version + ** of SQLite. */ + zNew = p->nAlloc<=0x7fffffff ? sqlite3_realloc(p->z, (int)p->nAlloc) : 0; if( zNew==0 ){ sqlite3_free(p->z); memset(p, 0, sizeof(*p)); @@ -93,7 +96,7 @@ static void sqlEvalFunc( sqlite3_result_error_nomem(context); sqlite3_free(x.z); }else{ - sqlite3_result_text(context, x.z, x.nUsed, sqlite3_free); + sqlite3_result_text(context, x.z, (int)x.nUsed, sqlite3_free); } } diff --git a/manifest b/manifest index 61039ed68d..96dcf87dd3 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Update\scomments\sin\ssqlite3ota.h\sto\sremove\sthe\s"must\shave\sPRIMARY\sKEY"\srestriction. -D 2014-12-08T07:28:26.753 +C Update\sthis\sbranch\swith\slatest\strunk\schanges. +D 2014-12-08T07:50:31.222 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb +F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 788f1288633a0c3c3cbbe0f3e4827d033f7ba530 +F Makefile.msc 10720782f88648bf2b5dcedf4c1524b067d43e47 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION d846487aff892625eb8e75960234e7285f0462fe @@ -86,7 +86,7 @@ F ext/fts3/fts3_expr.c 40123785eaa3ebd4c45c9b23407cc44ac0c49905 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 -F ext/fts3/fts3_porter.c 7f8b4bf5af7c0f20f73b8e87e14fa9298f52e290 +F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009 F ext/fts3/fts3_snippet.c 51beb5c1498176fd9caccaf1c75b55cb803a985a F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038 @@ -109,7 +109,7 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 678056a4bfcd83c4e82dea81d37543cd1d6dbee1 F ext/misc/closure.c 636024302cde41b2bf0c542f81c40c624cfb7012 F ext/misc/compress.c 76e45655f4046e756064ab10c62e18f2eb846b9f -F ext/misc/eval.c 04e630bde869aa1fec6b993d40591f963be2f868 +F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e @@ -182,13 +182,13 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c c59f238a39aacece176f8bb7dfece40deb268ee5 +F src/analyze.c 7a2986e6ea8247e5f21aca3d0b584598f58d84fe F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 1ab82a27ed4f1e8dc640b7ca0ffb074c4233e495 +F src/btree.c 7071995e9ab92173f43e9d1b8560a8db64a31e9a F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -197,7 +197,7 @@ F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 2d2c4ff24bda5d28000d0aeb05960ee2883a2d3a -F src/expr.c 73de4c0da2eed6b149d40a05c589dfeb2c4a87a1 +F src/expr.c 00da3072f362b06f39ce4052baa1d4ce2bb36d1c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c 6d3c4ebd72aa7923ce9b110a7dc15f9b8c548430 @@ -210,7 +210,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c cd819123ed552a15c37bd2fd5360db25015dc461 +F src/main.c eac81ee5cb0f94b496c15f20fcbabe4530b9a8c1 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -231,29 +231,29 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a9e500dd963fb1f67d7860e58b5772abe6123862 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 9fe27a768be53dbe0a818ae10791dc36100a0e76 +F src/pager.c 47f13c194a980ed55dd4825f286d40d49c4a7093 F src/pager.h c6157af66a9999797629968921133f67716f8f9f F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c facbdd3ecc09c8f750089d941305694301328e98 -F src/pragma.c 8e0087a5ae6e60ac9ed48df19025cb423e3c8c34 +F src/pragma.c 294c31d79dfcb6f9cea49528b19e5f8b25e3d5ec F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 +F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in f60a24616a6a7e622266e723ed141f0c6131514e +F src/select.c f377fb8a5c73c10678ea74f3400f7913943e3d75 +F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf +F src/sqlite.h.in 8f704473c8301f3c9cc044d10020bb3d5955dfc3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 32d7becef5cbd9a1118608921ec99f00f96c6e5d +F src/sqliteInt.h 9d7b1d5adfcc026971957d440b796f2b26b82d0c F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 8cf7d53aa1e1393b79457e4d49a29c18fa8403bd -F src/test1.c 76b5cbefaac443f48f05d3621f5cedc7b2801c65 +F src/test1.c fed17ded5498378fea274f2de0fa7b0b89f855f7 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -306,20 +306,20 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c ec1f55acef4864520ca2017b9f0d60c2ac1b8b78 +F src/vdbe.c 1a9e671c9cfc259e4d2affc71f7df4a4c00a842c F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 -F src/vdbeaux.c 5ce4f414147a3bc3cbcf00ec57f2606c25791629 +F src/vdbeaux.c 6f7f39c3fcf0f5923758df8561bb5d843908a553 F src/vdbeblob.c 317c71482ed73b0966db2d1c4e20839be3e9fe79 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a +F src/vdbesort.c 42c166f7ca78cb643c7f4e4bdfa83c59d363d1a6 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 -F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 -F src/wal.c d5c581b635951cf5513ec9699d118b32323443f3 +F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 +F src/wal.c 632d9afe19e11cc49a8b74ff52ec2a415568b958 F src/wal.h 0d3ba0c3f1b4c25796cb213568a84b9f9063f465 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c e275cb74731a3351a9da6ba8280bd5054db6192d +F src/where.c e914fdb9159bb36af4a673193bbda08aaf9e5a73 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -361,7 +361,7 @@ F test/auth2.test 264c6af53cad9aba5218c68bbe18036e39007bfa F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 -F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de +F test/autoindex2.test af7e595c6864cc6ef5fc38d5db579a3e34940cb8 F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972 F test/autoindex4.test fc807f9efd158bec60f5dfdf34ebe46fb274612d F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 @@ -381,6 +381,7 @@ F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 +F test/bigsort.test 835478d0ce83bd1e5b05c90571dedd9871a09196 F test/bind.test 3c7b320969000c441a70952b0b15938fbb66237c F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 @@ -393,6 +394,7 @@ F test/boundary3.tcl 23361e108a125dca9c4080c2feb884fe54d69243 F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b +F test/btree01.test e08b3613540145b353f20c81cb18ead54ff12e0f F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de @@ -467,7 +469,7 @@ F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 +F test/distinct.test 175d49ee783febaf368192dfe7f5afbc68910230 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_blobbytes.test 9bea1d3e2b20f3010b04abba58f6ba172301f49f F test/e_blobclose.test df756753f571bc30e42e3a6cba2807576e49e716 @@ -491,6 +493,7 @@ F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a +F test/e_walckpt.test de5a8d86c5b95569309c6da796dbea870c22e003 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 @@ -683,7 +686,7 @@ F test/join.test 52d4d49f86d0cf46926672878c4eaf0da399104a F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 -F test/join5.test 86675fc2919269aa923c84dd00ee4249b97990fe +F test/join5.test 5df23eba184f159ed9705a954957e765a10c141d F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 69abc726c51b4a0409189f9a85191205297c0577 F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 @@ -792,7 +795,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 19cc048448340952b08a897167ec7b7bc85142b9 +F test/permutations.test 7828a776c70fccf83d2e35d0e1efc191b3e0c646 F test/pragma.test 49ac8a73c0daa574824538fed28727d1259fe735 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -827,7 +830,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3 -F test/scanstatus.test a6dd739bc4d9638e8f5c2493b518057f2b681655 +F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -860,10 +863,11 @@ F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 00a07bf5e1337ecf72e94542bdefdc330d7a2538 F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 +F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test d60946b5fde4d85fe06db7331dfe89011f564350 -F test/shell2.test c57da3a381c099b02c813ba156298d5c2f5c93a3 +F test/shell1.test ab6025d941f9c84c5b83412c6b4d8b57f78dfa3a +F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29 F test/shell4.test 8a9c08976291e6c6c808b4d718f4a8b299f339f5 F test/shell5.test 15a419cc1df21c892ed64f5596ae7a501f2816f2 @@ -1105,7 +1109,7 @@ F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test f311691d696a5cc27e3c1b875cec1b0866b4ccd9 -F test/vtab1.test b631d147b198cfd7903ab5fed028eb2a3d321dc6 +F test/vtab1.test 1cef14310144718812351a61c5cfb4ba8494a171 F test/vtab2.test 7bcffc050da5c68f4f312e49e443063e2d391c0d F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e F test/vtab4.test 942f8b8280b3ea8a41dae20e7822d065ca1cb275 @@ -1127,7 +1131,7 @@ F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b +F test/wal5.test 174cc1512e304a7dfa28ac30527e28ea02fc37df F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd @@ -1168,7 +1172,7 @@ F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test 69f5ec4b15760a8c860f80e2d55525669390aab3 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test 1d89199697919d4930be05a71e7fe620f114e622 -F test/whereJ.test 63599653dfefe4e74ebb358db753417fe0aa8a49 +F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1237,7 +1241,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 46069393b3141ab198f0fcc4f6c05229f06bf1e8 -R 2ff45547a2c264d0b419c476d87c1792 +P 088a41eb8c18886a260cf53fa0cca3bd1958dc05 6aeece19a235344be2537e66a3fe08b1febfb5a0 +R 22c55b7623330ec4e5635caf9d27c83c U dan -Z 50a00d12069d2404879b86db79315481 +Z 93c0d69763bbc98aa3e78f69481d4c3f diff --git a/manifest.uuid b/manifest.uuid index 85a4152dfb..3420b8955c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -088a41eb8c18886a260cf53fa0cca3bd1958dc05 \ No newline at end of file +69a312ad3fe5b39bc394b9ce958cb63d734518c7 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index ffb5c22b63..e483807116 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1438,8 +1438,6 @@ static void decodeIntArray( if( z==0 ) z = ""; #else assert( z!=0 ); - pIndex->bUnordered = 0; - pIndex->noSkipScan = 0; #endif for(i=0; *z && ibUnordered = 1; - }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); - }else if( sqlite3_strglob("noskipscan*", z)==0 ){ - pIndex->noSkipScan = 1; - } + pIndex->bUnordered = 0; + pIndex->noSkipScan = 0; + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ + pIndex->bUnordered = 1; + }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ + pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + }else if( sqlite3_strglob("noskipscan*", z)==0 ){ + pIndex->noSkipScan = 1; + } #ifdef SQLITE_ENABLE_COSTMULT - else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ - pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); - } + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } #endif - while( z[0]!=0 && z[0]!=' ' ) z++; - while( z[0]==' ' ) z++; + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; + } } } @@ -1595,7 +1596,7 @@ static void initAvgEq(Index *pIdx){ i64 nSum100 = 0; /* Number of terms contributing to sumEq */ i64 nDist100; /* Number of distinct values in index */ - if( pIdx->aiRowEst==0 || pIdx->aiRowEst[iCol+1]==0 ){ + if( !pIdx->aiRowEst || iCol>=pIdx->nKeyCol || pIdx->aiRowEst[iCol+1]==0 ){ nRow = pFinal->anLt[iCol]; nDist100 = (i64)100 * pFinal->anDLt[iCol]; nSample--; diff --git a/src/btree.c b/src/btree.c index e7455ef396..5942085147 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1357,7 +1357,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ defragment_page: - testcase( pPage->nCell==0 ); + assert( pPage->nCell>0 || CORRUPT_DB ); rc = defragmentPage(pPage); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); @@ -6258,6 +6258,14 @@ static int pageFreeArray( } /* +** apCell[] and szCell[] contains pointers to and sizes of all cells in the +** pages being balanced. The current page, pPg, has pPg->nCell cells starting +** with apCell[iOld]. After balancing, this page should hold nNew cells +** starting at apCell[iNew]. +** +** This routine makes the necessary adjustments to pPg so that it contains +** the correct cells after being balanced. +** ** The pPg->nFree field is invalid when this function returns. It is the ** responsibility of the caller to set it correctly. */ @@ -6298,12 +6306,13 @@ static void editPage( ); } - pData = &aData[get2byte(&aData[hdr+5])]; + pData = &aData[get2byteNotZero(&aData[hdr+5])]; if( pDataaCellIdx; memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); if( pageInsertArray( @@ -6408,7 +6417,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( pPage->nOverflow==1 ); /* This error condition is now caught prior to reaching this function */ - if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; + if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell @@ -6851,7 +6860,11 @@ static int balance_nonroot( }else{ assert( leafCorrection==4 ); if( szCell[nCell]<4 ){ - /* Do not allow any cells smaller than 4 bytes. */ + /* Do not allow any cells smaller than 4 bytes. If a smaller cell + ** does exist, pad it with 0x00 bytes. */ + assert( szCell[nCell]==3 ); + assert( apCell[nCell]==&pTemp[iSpace1-3] ); + pTemp[iSpace1++] = 0x00; szCell[nCell] = 4; } } @@ -7086,7 +7099,11 @@ static int balance_nonroot( ** if sibling page iOld had the same page number as pNew, and if ** pCell really was a part of sibling page iOld (not a divider or ** overflow cell), we can skip updating the pointer map entries. */ - if( pNew->pgno!=aPgno[iOld] || pCell=&aOld[usableSize] ){ + if( iOld>=nNew + || pNew->pgno!=aPgno[iOld] + || pCell=&aOld[usableSize] + ){ if( !leafCorrection ){ ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); } diff --git a/src/expr.c b/src/expr.c index c28f196221..817975ab3a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1414,7 +1414,8 @@ int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: assert( p->pTab!=0 ); - return p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0; + return ExprHasProperty(p, EP_CanBeNull) || + (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0); default: return 1; } diff --git a/src/main.c b/src/main.c index 12dd56e2a1..9027a2a4a0 100644 --- a/src/main.c +++ b/src/main.c @@ -1032,6 +1032,14 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ for(j=0; jnDb; j++){ struct Db *pDb = &db->aDb[j]; if( pDb->pBt ){ + if( pDb->pSchema ){ + /* Must clear the KeyInfo cache. See ticket [e4a18565a36884b00edf] */ + for(i=sqliteHashFirst(&pDb->pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3KeyInfoUnref(pIdx->pKeyInfo); + pIdx->pKeyInfo = 0; + } + } sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; if( j!=1 ){ @@ -1936,10 +1944,13 @@ int sqlite3_wal_checkpoint_v2( if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; - assert( SQLITE_CHECKPOINT_FULL>SQLITE_CHECKPOINT_PASSIVE ); - assert( SQLITE_CHECKPOINT_FULLSQLITE_CHECKPOINT_RESTART ){ + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ return SQLITE_MISUSE; } @@ -1994,7 +2005,9 @@ int sqlite3_ckpt_open( ** checkpointed. */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ - return sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_PASSIVE, 0, 0); + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); } #ifndef SQLITE_OMIT_WAL diff --git a/src/pager.c b/src/pager.c index 3508a9f937..a5bba528ee 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7121,7 +7121,8 @@ int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; if( pPager->pWal && PagerOtaMode(pPager)==0 ){ rc = sqlite3WalCheckpoint(pPager->pWal, eMode, - pPager->xBusyHandler, pPager->pBusyHandlerArg, + (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + pPager->pBusyHandlerArg, pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); diff --git a/src/pragma.c b/src/pragma.c index 140d496abb..1eaa98ae29 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2257,7 +2257,7 @@ void sqlite3Pragma( #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [database.]wal_checkpoint = passive|full|restart + ** PRAGMA [database.]wal_checkpoint = passive|full|restart|truncate ** ** Checkpoint the database. */ @@ -2269,6 +2269,8 @@ void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_FULL; }else if( sqlite3StrICmp(zRight, "restart")==0 ){ eMode = SQLITE_CHECKPOINT_RESTART; + }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ + eMode = SQLITE_CHECKPOINT_TRUNCATE; } } sqlite3VdbeSetNumCols(v, 3); diff --git a/src/resolve.c b/src/resolve.c index 8fb580b3a1..d4bd548c93 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -320,6 +320,10 @@ static int lookupName( if( pMatch ){ pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; + assert( (pMatch->jointype & JT_RIGHT)==0 ); /* RIGHT JOIN not (yet) supported */ + if( (pMatch->jointype & JT_LEFT)!=0 ){ + ExprSetProperty(pExpr, EP_CanBeNull); + } pSchema = pExpr->pTab->pSchema; } } /* if( pSrcList ) */ diff --git a/src/select.c b/src/select.c index 3b422f1100..070ac00410 100644 --- a/src/select.c +++ b/src/select.c @@ -4829,7 +4829,7 @@ int sqlite3Select( ** ** is transformed to: ** - ** SELECT xyz FROM ... GROUP BY xyz + ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz ** ** The second form is preferred as a single index (or temp-table) may be ** used for both the ORDER BY and DISTINCT processing. As originally @@ -4842,7 +4842,6 @@ int sqlite3Select( p->selFlags &= ~SF_Distinct; p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; - sSort.pOrderBy = 0; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ diff --git a/src/shell.c b/src/shell.c index bbd37c7a04..94d20d3549 100644 --- a/src/shell.c +++ b/src/shell.c @@ -4038,10 +4038,12 @@ int main(int argc, char **argv){ char *zErrMsg = 0; ShellState data; const char *zInitFile = 0; - char *zFirstCmd = 0; int i; int rc = 0; int warnInmemoryDb = 0; + int readStdin = 1; + int nCmd = 0; + char **azCmd = 0; #if USE_SYSTEM_SQLITE+0!=1 if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ @@ -4061,6 +4063,18 @@ int main(int argc, char **argv){ signal(SIGINT, interrupt_handler); #endif +#ifdef SQLITE_SHELL_DBNAME_PROC + { + /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name + ** of a C-function that will provide the name of the database file. Use + ** this compile-time option to embed this shell program in larger + ** applications. */ + extern void SQLITE_SHELL_DBNAME_PROC(const char**); + SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); + warnInmemoryDb = 0; + } +#endif + /* Do an initial pass through the command-line argument to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, @@ -4072,15 +4086,18 @@ int main(int argc, char **argv){ if( z[0]!='-' ){ if( data.zDbFilename==0 ){ data.zDbFilename = z; - continue; + }else{ + /* Excesss arguments are interpreted as SQL (or dot-commands) and + ** mean that nothing is read from stdin */ + readStdin = 0; + nCmd++; + azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd); + if( azCmd==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } + azCmd[nCmd-1] = z; } - if( zFirstCmd==0 ){ - zFirstCmd = z; - continue; - } - fprintf(stderr,"%s: Error: too many options: \"%s\"\n", Argv0, argv[i]); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; } if( z[1]=='-' ) z++; if( strcmp(z,"-separator")==0 @@ -4170,11 +4187,6 @@ int main(int argc, char **argv){ #else fprintf(stderr,"%s: Error: no database filename specified\n", Argv0); return 1; -#endif -#ifdef SQLITE_SHELL_DBNAME_PROC - { extern void SQLITE_SHELL_DBNAME_PROC(const char**); - SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); - warnInmemoryDb = 0; } #endif } data.out = stdout; @@ -4272,6 +4284,10 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-help")==0 ){ usage(1); }else if( strcmp(z,"-cmd")==0 ){ + /* Run commands that follow -cmd first and separately from commands + ** that simply appear on the command-line. This seems goofy. It would + ** be better if all commands ran in the order that they appear. But + ** we retain the goofy behavior for historical compatibility. */ if( i==argc-1 ) break; z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ @@ -4295,23 +4311,28 @@ int main(int argc, char **argv){ } } - if( zFirstCmd ){ - /* Run just the command that follows the database name + if( !readStdin ){ + /* Run all arguments that do not begin with '-' as if they were separate + ** command-line inputs, except for the argToSkip argument which contains + ** the database filename. */ - if( zFirstCmd[0]=='.' ){ - rc = do_meta_command(zFirstCmd, &data); - if( rc==2 ) rc = 0; - }else{ - open_db(&data, 0); - rc = shell_exec(data.db, zFirstCmd, shell_callback, &data, &zErrMsg); - if( zErrMsg!=0 ){ - fprintf(stderr,"Error: %s\n", zErrMsg); - return rc!=0 ? rc : 1; - }else if( rc!=0 ){ - fprintf(stderr,"Error: unable to process SQL \"%s\"\n", zFirstCmd); - return rc; + for(i=0; i ** ** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as -** was given no the corresponding lock. +** was given on the corresponding lock. ** ** The xShmLock method can transition between unlocked and SHARED or ** between unlocked and EXCLUSIVE. It cannot transition between SHARED @@ -1524,8 +1524,8 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_MEMSTATUS]]
SQLITE_CONFIG_MEMSTATUS
**
^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, ** interpreted as a boolean, which enables or disables the collection of -** memory allocation statistics. ^(When memory allocation statistics are disabled, the -** following SQLite interfaces become non-operational: +** memory allocation statistics. ^(When memory allocation statistics are +** disabled, the following SQLite interfaces become non-operational: **
    **
  • [sqlite3_memory_used()] **
  • [sqlite3_memory_highwater()] @@ -1566,7 +1566,8 @@ struct sqlite3_mem_methods { ** This configuration should not be used if an application-define page ** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] ** configuration option. -** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to +** 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page ** (a power of two between 512 and 32768) plus some extra bytes for each @@ -1586,7 +1587,8 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    **
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer ** that SQLite will use for all of its dynamic memory allocation needs -** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. +** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and +** [SQLITE_CONFIG_PAGECACHE]. ** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled ** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns ** [SQLITE_ERROR] if invoked otherwise. @@ -1606,9 +1608,9 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_MUTEX]]
    SQLITE_CONFIG_MUTEX
    **
    ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a ** pointer to an instance of the [sqlite3_mutex_methods] structure. -** The argument specifies alternative low-level mutex routines to be used in place -** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the -** content of the [sqlite3_mutex_methods] structure before the call to +** The argument specifies alternative low-level mutex routines to be used +** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then ** the entire mutexing subsystem is omitted from the build and hence calls to @@ -1646,8 +1648,8 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    **
    ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which -** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of the current -** page cache implementation into that object.)^
    +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** the current page cache implementation into that object.)^
** ** [[SQLITE_CONFIG_LOG]]
SQLITE_CONFIG_LOG
**
The SQLITE_CONFIG_LOG option is used to configure the SQLite @@ -1672,8 +1674,9 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_URI]]
SQLITE_CONFIG_URI **
^(The SQLITE_CONFIG_URI option takes a single argument of type int. ** If non-zero, then URI handling is globally enabled. If the parameter is zero, -** then URI handling is globally disabled.)^ ^If URI handling is globally enabled, -** all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +** then URI handling is globally disabled.)^ ^If URI handling is globally +** enabled, all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], +** [sqlite3_open16()] or ** specified as part of [ATTACH] commands are interpreted as URIs, regardless ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database ** connection is opened. ^If it is globally disabled, filenames are @@ -1735,8 +1738,8 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] **
SQLITE_CONFIG_WIN32_HEAPSIZE **
^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is -** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. -** ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro +** defined. ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** ** @@ -1744,8 +1747,8 @@ struct sqlite3_mem_methods { **
SQLITE_CONFIG_PCACHE_HDRSZ **
^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which ** is a pointer to an integer and writes into that integer the number of extra -** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. The amount of -** extra space required can change depending on the compiler, +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. +** The amount of extra space required can change depending on the compiler, ** target platform, and SQLite version. ** */ @@ -2049,6 +2052,7 @@ int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** KEYWORDS: {busy-handler callback} {busy handler} ** ** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X ** that might be invoked with argument P whenever @@ -2065,7 +2069,7 @@ int sqlite3_complete16(const void *sql); ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for the same locking event. ^If the +** been invoked previously for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to ** access the database and [SQLITE_BUSY] is returned ** to the application. @@ -4520,7 +4524,8 @@ typedef void (*sqlite3_destructor_type)(void*); ** the [sqlite3_context] pointer, the results are undefined. */ void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); -void sqlite3_result_blob64(sqlite3_context*,const void*,sqlite3_uint64,void(*)(void*)); +void sqlite3_result_blob64(sqlite3_context*,const void*, + sqlite3_uint64,void(*)(void*)); void sqlite3_result_double(sqlite3_context*, double); void sqlite3_result_error(sqlite3_context*, const char*, int); void sqlite3_result_error16(sqlite3_context*, const void*, int); @@ -7246,97 +7251,114 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); /* ** CAPI3REF: Checkpoint a database ** -** ^The [sqlite3_wal_checkpoint(D,X)] interface causes database named X -** on [database connection] D to be [checkpointed]. ^If X is NULL or an -** empty string, then a checkpoint is run on all databases of -** connection D. ^If the database connection D is not in -** [WAL | write-ahead log mode] then this interface is a harmless no-op. -** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a -** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. -** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL -** or RESET checkpoint. +** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to +** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ ** -** ^The [wal_checkpoint pragma] can be used to invoke this interface -** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] can be used to cause this interface to be -** run whenever the WAL reaches a certain size threshold. +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** [write-ahead log] for database X on [database connection] D to be +** transferred into the database file and for the write-ahead log to +** be reset. See the [checkpointing] documentation for addition +** information. ** -** See also: [sqlite3_wal_checkpoint_v2()] +** This interface used to be the only way to cause a checkpoint to +** occur. But then the newer and more powerful [sqlite3_wal_checkpoint_v2()] +** interface was added. This interface is retained for backwards +** compatibility and as a convenience for applications that need to manually +** start a callback but which do not need the full power (and corresponding +** complication) of [sqlite3_wal_checkpoint_v2()]. */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Checkpoint a database ** -** Run a checkpoint operation on WAL database zDb attached to database -** handle db. The specific operation is determined by the value of the -** eMode parameter: +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint +** operation on database X of [database connection] D in mode M. Status +** information is written back into integers pointed to by L and C.)^ +** ^(The M parameter must be a valid [checkpoint mode]:)^ ** **
**
SQLITE_CHECKPOINT_PASSIVE
-** Checkpoint as many frames as possible without waiting for any database -** readers or writers to finish. Sync the db file if all frames in the log -** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] -** is never invoked. +** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames +** in the log were checkpointed. ^The [busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** ^On the other hand, passive mode might leave the checkpoint unfinished +** if there are concurrent readers or writers. ** **
SQLITE_CHECKPOINT_FULL
-** This mode blocks (it invokes the +** ^This mode blocks (it invokes the ** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database -** snapshot. It then checkpoints all frames in the log file and syncs the -** database file. This call blocks database writers while it is running, -** but not database readers. +** snapshot. ^It then checkpoints all frames in the log file and syncs the +** database file. ^This mode blocks new database writers while it is pending, +** but new database readers are allowed to continue unimpeded. ** **
SQLITE_CHECKPOINT_RESTART
-** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the -** [sqlite3_busy_handler|busy-handler callback]) -** until all readers are reading from the database file only. This ensures -** that the next client to write to the database file restarts the log file -** from the beginning. This call blocks database writers while it is running, -** but not database readers. +** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition +** that after checkpointing the log file it blocks (calls the +** [busy-handler callback]) +** until all readers are reading from the database file only. ^This ensures +** that the next writer will restart the log file from the beginning. +** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new +** database writer attempts while it is pending, but does not impede readers. +** +**
SQLITE_CHECKPOINT_TRUNCATE
+** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the +** addition that it also truncates the log file to zero bytes just prior +** to a successful return. **
** -** If pnLog is not NULL, then *pnLog is set to the total number of frames in -** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to -** the total number of checkpointed frames (including any that were already -** checkpointed when this function is called). *pnLog and *pnCkpt may be -** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. -** If no values are available because of an error, they are both set to -1 -** before returning to communicate this to the caller. +** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file or to -1 if the checkpoint could not run because +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not +** NULL,then *pnCkpt is set to the total number of checkpointed frames in the +** log file (including any that were already checkpointed before the function +** was called) or to -1 if the checkpoint could not run due to an error or +** because the database is not in WAL mode. ^Note that upon successful +** completion of an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been +** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. ** -** All calls obtain an exclusive "checkpoint" lock on the database file. If +** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If ** any other process is running a checkpoint operation at the same time, the -** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a ** busy-handler configured, it will not be invoked in this case. ** -** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive -** "writer" lock on the database file. If the writer lock cannot be obtained -** immediately, and a busy-handler is configured, it is invoked and the writer -** lock retried until either the busy-handler returns 0 or the lock is -** successfully obtained. The busy-handler is also invoked while waiting for -** database readers as described above. If the busy-handler returns 0 before +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. ^If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. ^The busy-handler is also invoked while waiting for +** database readers as described above. ^If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the ** checkpoint operation proceeds from that point in the same way as ** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible -** without blocking any further. SQLITE_BUSY is returned in this case. +** without blocking any further. ^SQLITE_BUSY is returned in this case. ** -** If parameter zDb is NULL or points to a zero length string, then the -** specified operation is attempted on all WAL databases. In this case the -** values written to output parameters *pnLog and *pnCkpt are undefined. If +** ^If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases [attached] to +** [database connection] db. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If ** an SQLITE_BUSY error is encountered when processing one or more of the ** attached WAL databases, the operation is still attempted on any remaining -** attached databases and SQLITE_BUSY is returned to the caller. If any other +** attached databases and SQLITE_BUSY is returned at the end. ^If any other ** error occurs while processing an attached database, processing is abandoned -** and the error code returned to the caller immediately. If no error +** and the error code is returned to the caller immediately. ^If no error ** (SQLITE_BUSY or otherwise) is encountered while processing the attached ** databases, SQLITE_OK is returned. ** -** If database zDb is the name of an attached database that is not in WAL -** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** ^If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. ^If ** zDb is not NULL (or a zero length string) and is not the name of any ** attached database, SQLITE_ERROR is returned to the caller. +** +** ^Unless it returns SQLITE_MISUSE, +** the sqlite3_wal_checkpoint_v2() interface +** sets the error information that is queried by +** [sqlite3_errcode()] and [sqlite3_errmsg()]. +** +** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface +** from SQL. */ int sqlite3_wal_checkpoint_v2( sqlite3 *db, /* Database handle */ @@ -7347,16 +7369,18 @@ int sqlite3_wal_checkpoint_v2( ); /* -** CAPI3REF: Checkpoint operation parameters +** CAPI3REF: Checkpoint Mode Values +** KEYWORDS: {checkpoint mode} ** -** These constants can be used as the 3rd parameter to -** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] -** documentation for additional information about the meaning and use of -** each of these values. +** These constants define all valid values for the "checkpoint mode" passed +** as the third parameter to the [sqlite3_wal_checkpoint_v2()] interface. +** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the +** meaning of each of these checkpoint modes. */ -#define SQLITE_CHECKPOINT_PASSIVE 0 -#define SQLITE_CHECKPOINT_FULL 1 -#define SQLITE_CHECKPOINT_RESTART 2 +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* ** CAPI3REF: Virtual Table Interface Configuration @@ -7455,12 +7479,12 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** **
** [[SQLITE_SCANSTAT_NLOOP]]
SQLITE_SCANSTAT_NLOOP
-**
^The [sqlite3_int64] variable pointed to by the T parameter will be set to the -** total number of times that the X-th loop has run.
+**
^The [sqlite3_int64] variable pointed to by the T parameter will be +** set to the total number of times that the X-th loop has run.
** ** [[SQLITE_SCANSTAT_NVISIT]]
SQLITE_SCANSTAT_NVISIT
-**
^The [sqlite3_int64] variable pointed to by the T parameter will be set to the -** total number of rows examined by all iterations of the X-th loop.
+**
^The [sqlite3_int64] variable pointed to by the T parameter will be set +** to the total number of rows examined by all iterations of the X-th loop.
** ** [[SQLITE_SCANSTAT_EST]]
SQLITE_SCANSTAT_EST
**
^The "double" variable pointed to by the T parameter will be set to the @@ -7471,14 +7495,14 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** be the NLOOP value for the current loop. ** ** [[SQLITE_SCANSTAT_NAME]]
SQLITE_SCANSTAT_NAME
-**
^The "const char *" variable pointed to by the T parameter will be set to -** a zero-terminated UTF-8 string containing the name of the index or table used -** for the X-th loop. +**
^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the name of the index or table +** used for the X-th loop. ** ** [[SQLITE_SCANSTAT_EXPLAIN]]
SQLITE_SCANSTAT_EXPLAIN
-**
^The "const char *" variable pointed to by the T parameter will be set to -** a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] description -** for the X-th loop. +**
^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] +** description for the X-th loop. ** ** [[SQLITE_SCANSTAT_SELECTID]]
SQLITE_SCANSTAT_SELECT
**
^The "int" variable pointed to by the T parameter will be set to the @@ -7501,8 +7525,8 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** Return status data for a single loop within query pStmt. ** ** The "iScanStatusOp" parameter determines which status information to return. -** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior of -** this interface is undefined. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior +** of this interface is undefined. ** ^The requested measurement is written into a variable pointed to by ** the "pOut" parameter. ** Parameter "idx" identifies the specific loop to retrieve statistics for. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b89214354e..40bdbe5d3f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2020,7 +2020,7 @@ struct Expr { /* ** The following are the meanings of bits in the Expr.flags field. */ -#define EP_FromJoin 0x000001 /* Originated in ON or USING clause of a join */ +#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ #define EP_Agg 0x000002 /* Contains one or more aggregate functions */ #define EP_Resolved 0x000004 /* IDs have been resolved to COLUMNs */ #define EP_Error 0x000008 /* Expression contains one or more errors */ @@ -2040,6 +2040,7 @@ struct Expr { #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ #define EP_Constant 0x080000 /* Node is a constant */ +#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ /* ** These macros can be used to test, set, or clear bits in the diff --git a/src/test1.c b/src/test1.c index 272d708044..fb601badff 100644 --- a/src/test1.c +++ b/src/test1.c @@ -3638,6 +3638,7 @@ static int test_prepare_v2( ){ sqlite3 *db; const char *zSql; + char *zCopy = 0; /* malloc() copy of zSql */ int bytes; const char *zTail = 0; sqlite3_stmt *pStmt = 0; @@ -3653,7 +3654,21 @@ static int test_prepare_v2( zSql = Tcl_GetString(objv[2]); if( Tcl_GetIntFromObj(interp, objv[3], &bytes) ) return TCL_ERROR; - rc = sqlite3_prepare_v2(db, zSql, bytes, &pStmt, objc>=5 ? &zTail : 0); + /* Instead of using zSql directly, make a copy into a buffer obtained + ** directly from malloc(). The idea is to make it easier for valgrind + ** to spot buffer overreads. */ + if( bytes>=0 ){ + zCopy = malloc(bytes); + memcpy(zCopy, zSql, bytes); + }else{ + int n = (int)strlen(zSql) + 1; + zCopy = malloc(n); + memcpy(zCopy, zSql, n); + } + rc = sqlite3_prepare_v2(db, zCopy, bytes, &pStmt, objc>=5 ? &zTail : 0); + free(zCopy); + zTail = &zSql[(zTail - zCopy)]; + assert(rc==SQLITE_OK || pStmt==0); Tcl_ResetResult(interp); if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; @@ -5675,10 +5690,11 @@ static int test_wal_checkpoint_v2( int nCkpt = -555; Tcl_Obj *pRet; - const char * aMode[] = { "passive", "full", "restart", 0 }; + const char * aMode[] = { "passive", "full", "restart", "truncate", 0 }; assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); if( objc!=3 && objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB MODE ?NAME?"); @@ -5688,15 +5704,17 @@ static int test_wal_checkpoint_v2( if( objc==4 ){ zDb = Tcl_GetString(objv[3]); } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) - || Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) - ){ + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) || ( + TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eMode) + && TCL_OK!=Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) + )){ return TCL_ERROR; } rc = sqlite3_wal_checkpoint_v2(db, zDb, eMode, &nLog, &nCkpt); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ - Tcl_SetResult(interp, (char *)sqlite3_errmsg(db), TCL_VOLATILE); + const char *zErrCode = sqlite3ErrName(rc); + Tcl_AppendResult(interp, zErrCode, " - ", (char *)sqlite3_errmsg(db), 0); return TCL_ERROR; } diff --git a/src/vdbe.c b/src/vdbe.c index 822bf80bb8..3dac74dfd4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5705,8 +5705,8 @@ case OP_AggFinal: { /* Opcode: Checkpoint P1 P2 P3 * * ** ** Checkpoint database P1. This is a no-op if P1 is not currently in -** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL -** or RESTART. Write 1 or 0 into mem[P3] if the checkpoint returns +** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL, +** RESTART, or TRUNCATE. Write 1 or 0 into mem[P3] if the checkpoint returns ** SQLITE_BUSY or not, respectively. Write the number of pages in the ** WAL after the checkpoint into mem[P3+1] and the number of pages ** in the WAL that have been checkpointed after the checkpoint @@ -5724,6 +5724,7 @@ case OP_Checkpoint: { assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART + || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc==SQLITE_BUSY ){ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index b8f5bca2c3..7900bd52ac 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1742,7 +1742,7 @@ void sqlite3VdbeMakeReady( p->aVar[n].db = db; } } - if( p->azVar ){ + if( p->azVar && pParse->nzVar>0 ){ p->nzVar = pParse->nzVar; memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); diff --git a/src/vdbesort.c b/src/vdbesort.c index df8357a57e..7c736adefe 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -147,6 +147,13 @@ # define SQLITE_DEBUG_SORTER_THREADS 1 #endif +/* +** Hard-coded maximum amount of data to accumulate in memory before flushing +** to a level 0 PMA. The purpose of this limit is to prevent various integer +** overflows. 512MiB. +*/ +#define SQLITE_MAX_MXPMASIZE (1<<29) + /* ** Private objects used by the sorter */ @@ -845,7 +852,7 @@ int sqlite3VdbeSorterInit( pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; mxCache = db->aDb[0].pSchema->cache_size; if( mxCachemxPmaSize = mxCache * pgsz; + pSorter->mxPmaSize = MIN((i64)mxCache*pgsz, SQLITE_MAX_MXPMASIZE); /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary diff --git a/src/vtab.c b/src/vtab.c index 334de9aacb..00d0882b04 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -332,7 +332,12 @@ void sqlite3VtabBeginParse( addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); - pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) + || (pParse->sNameToken.z==pName1->z && pName2->z==0) + ); + pParse->sNameToken.n = (int)( + &pModuleName->z[pModuleName->n] - pParse->sNameToken.z + ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Creating a virtual table invokes the authorization callback twice. diff --git a/src/wal.c b/src/wal.c index bb8c0f806e..726afbeb2a 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1829,6 +1829,38 @@ static int walCheckpointFinalize(WalCkpt *p){ return p->rc; } +/* +** The following is guaranteed when this function is called: +** +** a) the WRITER lock is held, +** b) the entire log file has been checkpointed, and +** c) any existing readers are reading exclusively from the database +** file - there are no readers that may attempt to read a frame from +** the log file. +** +** This function updates the shared-memory structures so that the next +** client to write to the database (which may be this one) does so by +** writing frames into the start of the log file. +** +** The value of parameter salt1 is used as the aSalt[1] value in the +** new wal-index header. It should be passed a pseudo-random value (i.e. +** one obtained from sqlite3_randomness()). +*/ +static void walRestartHdr(Wal *pWal, u32 salt1){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int i; /* Loop counter */ + u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + pWal->nCkpt++; + pWal->hdr.mxFrame = 0; + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + memcpy(&pWal->hdr.aSalt[1], &salt1, 4); + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + pInfo->aReadMark[1] = 0; + for(i=2; iaReadMark[i] = READMARK_NOT_USED; + assert( pInfo->aReadMark[0]==0 ); +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -1863,17 +1895,19 @@ static int walCheckpointFinalize(WalCkpt *p){ static int walCheckpoint( Wal *pWal, /* Wal connection */ int eMode, /* One of PASSIVE, FULL or RESTART */ - int (*xBusyCall)(void*), /* Function to call when busy */ + int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ u8 *zBuf, /* Temporary buffer to use */ int nBuf /* Size of zBuf in bytes */ ){ int rc; /* Return code */ - int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ WalCkpt sC; - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + rc = walCheckpointStart(pWal, zBuf, nBuf, xBusy, pBusyArg, sync_flags, &sC); if( sC.pIter==0 ) goto walcheckpoint_out; assert( rc==SQLITE_OK ); @@ -1883,19 +1917,38 @@ static int walCheckpoint( while( SQLITE_OK==(rc = walCheckpointStep(&sC)) ); rc = walCheckpointFinalize(&sC); - /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal - ** file has been copied into the database file, then block until all - ** readers have finished using the wal file. This ensures that the next - ** process to write to the database restarts the wal file. + /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the + ** entire wal file has been copied into the database file, then block + ** until all readers have finished using the wal file. This ensures that + ** the next process to write to the database restarts the wal file. */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( sC.pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; - }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ + }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ + u32 salt1; + sqlite3_randomness(4, &salt1); assert( sC.mxSafeFrame==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ + if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ + /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as + ** SQLITE_CHECKPOINT_RESTART with the addition that it also + ** truncates the log file to zero bytes just prior to a + ** successful return. + ** + ** In theory, it might be safe to do this without updating the + ** wal-index header in shared memory, as all subsequent reader or + ** writer clients should see that the entire log file has been + ** checkpointed and behave accordingly. This seems unsafe though, + ** as it would leave the system in a state where the contents of + ** the wal-index header do not match the contents of the + ** file-system. To avoid this, update the wal-index header to + ** indicate that the log file contains zero valid frames. */ + walRestartHdr(pWal, salt1); + rc = sqlite3OsTruncate(pWal->pWalFd, 0); + } walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); } } @@ -2689,7 +2742,6 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ return rc; } - /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending @@ -2722,20 +2774,8 @@ static int walRestartLog(Wal *pWal){ ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() - ** to handle if this transaction is rolled back. - */ - int i; /* Loop counter */ - u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ - - pWal->nCkpt++; - pWal->hdr.mxFrame = 0; - sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); - aSalt[1] = salt1; - walIndexWriteHdr(pWal); - pInfo->nBackfill = 0; - pInfo->aReadMark[1] = 0; - for(i=2; iaReadMark[i] = READMARK_NOT_USED; - assert( pInfo->aReadMark[0]==0 ); + ** to handle if this transaction is rolled back. */ + walRestartHdr(pWal, salt1); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); }else if( rc!=SQLITE_BUSY ){ return rc; @@ -3025,7 +3065,7 @@ int sqlite3WalFrames( */ int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ - int eMode, /* PASSIVE, FULL or RESTART */ + int eMode, /* PASSIVE, FULL, RESTART, or TRUNCATE */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags to sync db file with (or 0) */ @@ -3037,29 +3077,42 @@ int sqlite3WalCheckpoint( int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ + int (*xBusy2)(void*) = xBusy; /* Busy handler for eMode2 */ assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + + /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive + ** "checkpoint" lock on the database file. */ rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ - /* Usually this is SQLITE_BUSY meaning that another thread or process - ** is already running a checkpoint, or maybe a recovery. But it might - ** also be SQLITE_IOERR. */ + /* EVIDENCE-OF: R-10421-19736 If any other process is running a + ** checkpoint operation at the same time, the lock cannot be obtained and + ** SQLITE_BUSY is returned. + ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + ** it will not be invoked in this case. + */ + testcase( rc==SQLITE_BUSY ); + testcase( xBusy!=0 ); return rc; } pWal->ckptLock = 1; - /* If this is a blocking-checkpoint, then obtain the write-lock as well - ** to prevent any writers from running while the checkpoint is underway. - ** This has to be done before the call to walIndexReadHdr() below. + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + ** TRUNCATE modes also obtain the exclusive "writer" lock on the database + ** file. ** - ** If the writer lock cannot be obtained, then a passive checkpoint is - ** run instead. Since the checkpointer is not holding the writer lock, - ** there is no point in blocking waiting for any readers. Assuming no - ** other error occurs, this function will return SQLITE_BUSY to the caller. + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. */ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); @@ -3067,6 +3120,7 @@ int sqlite3WalCheckpoint( pWal->writeLock = 1; }else if( rc==SQLITE_BUSY ){ eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; rc = SQLITE_OK; } } @@ -3081,7 +3135,7 @@ int sqlite3WalCheckpoint( /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ - rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf, nBuf); + rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf, nBuf); /* If no error occurred, set the output variables. */ if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ diff --git a/src/where.c b/src/where.c index 8c9ce38889..ee42534f34 100644 --- a/src/where.c +++ b/src/where.c @@ -222,6 +222,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ sqlite3DbFree(db, pOld); } pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm)); } pTerm = &pWC->a[idx = pWC->nTerm++]; if( p && ExprHasProperty(p, EP_Unlikely) ){ @@ -2947,7 +2948,7 @@ static void addScanStatus( ){ const char *zObj = 0; WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ zObj = pLoop->u.btree.pIndex->zName; }else{ zObj = pSrclist->a[pLvl->iFrom].zName; @@ -3591,10 +3592,9 @@ static Bitmask codeOneLoopStart( Expr *pExpr = pWC->a[iTerm].pExpr; if( &pWC->a[iTerm] == pTerm ) continue; if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); - testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); - if( pWC->a[iTerm].wtFlags & (TERM_ORINFO|TERM_VIRTUAL) ) continue; + if( (pWC->a[iTerm].wtFlags & TERM_VIRTUAL)!=0 ) continue; if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); pExpr = sqlite3ExprDup(db, pExpr, 0); pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); } @@ -4291,10 +4291,30 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** Adjust the WhereLoop.nOut value downward to account for terms of the ** WHERE clause that reference the loop but which are not used by an ** index. +* +** For every WHERE clause term that is not used by the index +** and which has a truth probability assigned by one of the likelihood(), +** likely(), or unlikely() SQL functions, reduce the estimated number +** of output rows by the probability specified. ** -** In the current implementation, the first extra WHERE clause term reduces -** the number of output rows by a factor of 10 and each additional term -** reduces the number of output rows by sqrt(2). +** TUNING: For every WHERE clause term that is not used by the index +** and which does not have an assigned truth probability, heuristics +** described below are used to try to estimate the truth probability. +** TODO --> Perhaps this is something that could be improved by better +** table statistics. +** +** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% +** value corresponds to -1 in LogEst notation, so this means decrement +** the WhereLoop.nOut field for every such WHERE clause term. +** +** Heuristic 2: If there exists one or more WHERE clause terms of the +** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the +** final output row estimate is no greater than 1/4 of the total number +** of rows in the table. In other words, assume that x==EXPR will filter +** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the +** "x" column is boolean or else -1 or 0 or 1 is a common default value +** on the "x" column and so in that case only cap the output row estimate +** at 1/2 instead of 1/4. */ static void whereLoopOutputAdjust( WhereClause *pWC, /* The WHERE clause */ @@ -4303,9 +4323,10 @@ static void whereLoopOutputAdjust( ){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); - int i, j; - int nEq = 0; /* Number of = constraints not within likely()/unlikely() */ + int i, j, k; + LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ + assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; @@ -4318,20 +4339,26 @@ static void whereLoopOutputAdjust( } if( j<0 ){ if( pTerm->truthProb<=0 ){ + /* If a truth probability is specified using the likelihood() hints, + ** then use the probability provided by the application. */ pLoop->nOut += pTerm->truthProb; }else{ + /* In the absence of explicit truth probabilities, use heuristics to + ** guess a reasonable truth probability. */ pLoop->nOut--; - if( pTerm->eOperator&WO_EQ ) nEq++; + if( pTerm->eOperator&WO_EQ ){ + Expr *pRight = pTerm->pExpr->pRight; + if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + k = 10; + }else{ + k = 20; + } + if( iReducenOut>nRow-10 ){ - pLoop->nOut = nRow - 10; - } + if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; } /* diff --git a/test/autoindex2.test b/test/autoindex2.test index f4da416745..c8abdf410a 100644 --- a/test/autoindex2.test +++ b/test/autoindex2.test @@ -224,48 +224,4 @@ do_execsql_test autoindex2-120 { # on t3 and reordering the tables so that t3 was in the outer loop and # implementing the ORDER BY clause using a B-Tree. -do_execsql_test autoindex2-120 { - EXPLAIN QUERY PLAN - SELECT - t1_id, - t1.did, - param2, - param3, - t1.ptime, - t1.trange, - t1.exmass, - t1.mass, - t1.vstatus, - type, - subtype, - t1.deviation, - t1.formula, - dparam1, - reserve1, - reserve2, - param4, - t1.last_operation, - t1.admin_uuid, - t1.previous_value, - t1.job_id, - client_did, - t1.last_t1, - t1.data_t1, - t1.previous_date, - param5, - param6, - mgr_uuid - FROM - t3, - t2, - t1 - WHERE - t1.ptime > 1393520400 - AND param3<>9001 - AND t3.flg7 = 1 - AND t1.did = t2.did - AND t2.uid = t3.uid - ORDER BY t1.ptime desc LIMIT 500; -} {0 0 2 {SEARCH TABLE t1 USING INDEX t1x1 (ptime>?)} 0 1 1 {SEARCH TABLE t2 USING INDEX t2x0 (did=?)} 0 2 0 {SEARCH TABLE t3 USING INDEX t3x0 (uid=?)}} - finish_test diff --git a/test/bigsort.test b/test/bigsort.test new file mode 100644 index 0000000000..259adc3747 --- /dev/null +++ b/test/bigsort.test @@ -0,0 +1,43 @@ +# 2014 November 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bigsort + +#-------------------------------------------------------------------- +# At one point there was an overflow problem if the product of the +# cache-size and page-size was larger than 2^31. Causing an infinite +# loop if the product was also an integer multiple of 2^32, or +# inefficiency otherwise. +# +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + BEGIN; + WITH data(x,y) AS ( + SELECT 1, zeroblob(10000) + UNION ALL + SELECT x+1, y FROM data WHERE x < 300000 + ) + INSERT INTO t1 SELECT * FROM data; + COMMIT; +} +do_execsql_test 1.1 { + PRAGMA cache_size = 4194304; + CREATE INDEX i1 ON t1(a, b); +} + + +finish_test + + diff --git a/test/btree01.test b/test/btree01.test new file mode 100644 index 0000000000..25f2c6897b --- /dev/null +++ b/test/btree01.test @@ -0,0 +1,132 @@ +# 2014-11-27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains test cases for b-tree logic. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix btree01 + +# The refactoring on the b-tree balance() routine in check-in +# http://www.sqlite.org/src/info/face33bea1ba3a (2014-10-27) +# caused the integrity_check on the following SQL to fail. +# +do_execsql_test btree01-1.1 { + PRAGMA page_size=65536; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(3000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=2; + PRAGMA integrity_check; +} {ok} + +# The previous test is sufficient to prevent a regression. But we +# add a number of additional tests to stress the balancer in similar +# ways, looking for related problems. +# +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.2.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(3000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.3.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(2000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.4.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==0; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==1; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==2; + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.5.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6542) FROM c; + UPDATE t1 SET b=zeroblob(2331); + UPDATE t1 SET b=zeroblob(65496) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.6.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6542) FROM c; + UPDATE t1 SET b=zeroblob(2332); + UPDATE t1 SET b=zeroblob(65496) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.7.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(1); + UPDATE t1 SET b=zeroblob(65000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=31} {incr i} { + do_test btree01-1.8.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<31) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(4000); + UPDATE t1 SET b=zeroblob(65000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} + +finish_test diff --git a/test/distinct.test b/test/distinct.test index 78c2c1df36..2fb90dc3e3 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -222,4 +222,34 @@ do_execsql_test 4.1 { SELECT quote(x) FROM t2 ORDER BY 1; } {'xyzzy' X'0000000000'} +#---------------------------------------------------------------------------- +# Ticket [c5ea805691bfc4204b1cb9e9aa0103bd48bc7d34] (2014-12-04) +# Make sure that DISTINCT works together with ORDER BY and descending +# indexes. +# +do_execsql_test 5.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES(3),(1),(5),(2),(6),(4),(5),(1),(3); + CREATE INDEX t1x ON t1(x DESC); + SELECT DISTINCT x FROM t1 ORDER BY x ASC; +} {1 2 3 4 5 6} +do_execsql_test 5.2 { + SELECT DISTINCT x FROM t1 ORDER BY x DESC; +} {6 5 4 3 2 1} +do_execsql_test 5.3 { + SELECT DISTINCT x FROM t1 ORDER BY x; +} {1 2 3 4 5 6} +do_execsql_test 5.4 { + DROP INDEX t1x; + CREATE INDEX t1x ON t1(x ASC); + SELECT DISTINCT x FROM t1 ORDER BY x ASC; +} {1 2 3 4 5 6} +do_execsql_test 5.5 { + SELECT DISTINCT x FROM t1 ORDER BY x DESC; +} {6 5 4 3 2 1} +do_execsql_test 5.6 { + SELECT DISTINCT x FROM t1 ORDER BY x; +} {1 2 3 4 5 6} + finish_test diff --git a/test/e_walckpt.test b/test/e_walckpt.test new file mode 100644 index 0000000000..8b0aae798a --- /dev/null +++ b/test/e_walckpt.test @@ -0,0 +1,290 @@ +# 2014 December 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set testprefix e_walckpt + +# The following two commands are used to determine if any of the files +# "test.db", "test.db2" and "test.db3" are modified by a test case. +# +# The [save_db_hashes] command saves a hash of the current contents of +# all three files in global variables. The [compare_db_hashes] compares +# the current contents with the saved hashes and returns a list of the +# files that have changed. +# +proc save_db_hashes {} { + global H + foreach f {test.db test.db2 test.db3} { + set H($f) 0 + catch { set H($f) [md5file $f] } + } +} +proc compare_db_hashes {} { + global H + set ret [list] + foreach f {test.db test.db2 test.db3} { + set expect 0 + catch { set expect [md5file $f] } + if {$H($f) != $expect} { lappend ret $f } + } + set ret +} + + +# The following tests are run 3 times, each using a different method of +# invoking a checkpoint: +# +# 1) Using sqlite3_wal_checkpoint_v2() +# 2) Using "PRAGMA wal_checkpoint" +# 3) Using sqlite3_wal_checkpoint() in place of checkpoint_v2(PASSIVE) +# +# Cases (2) and (3) are to show that the following statements are +# correct, respectively: +# +# EVIDENCE-OF: R-36706-10507 The PRAGMA wal_checkpoint command can be +# used to invoke this interface from SQL. +# +# EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is +# equivalent to +# sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). +# +foreach {tn script} { + 1 { + proc checkpoint {db mode args} { + eval sqlite3_wal_checkpoint_v2 [list $db] [list $mode] $args + } + } + + 2 { + proc checkpoint {db mode args} { + set sql "PRAGMA wal_checkpoint" + if {[llength $args] && [lindex $args 0]!=""} { + set sql "PRAGMA [lindex $args 0].wal_checkpoint" + } + set rc [catch { $db eval $sql } msg] + if {$rc} { + regsub {database} $msg {database:} msg + error "[sqlite3_errcode $db] - $msg" + } + set msg + } + } + + 3 { + proc checkpoint {db mode args} { + if {$mode == "passive"} { + set rc [eval sqlite3_wal_checkpoint [list $db] $args] + if {$rc != "SQLITE_OK"} { + error "$rc - [sqlite3_errmsg $db]" + } + } else { + eval sqlite3_wal_checkpoint_v2 [list $db] [list $mode] $args + } + } + } + +} { + + eval $script + + reset_db + forcedelete test.db2 test.db3 test.db4 + execsql { + ATTACH 'test.db2' AS aux; + ATTACH 'test.db3' AS aux2; + ATTACH 'test.db4' AS aux3; + CREATE TABLE t1(x); + CREATE TABLE aux.t2(x); + CREATE TABLE aux2.t3(x); + CREATE TABLE aux3.t4(x); + PRAGMA main.journal_mode = WAL; + PRAGMA aux.journal_mode = WAL; + PRAGMA aux2.journal_mode = WAL; + /* Leave aux4 in rollback mode */ + } + + # EVIDENCE-OF: R-49787-09095 The sqlite3_wal_checkpoint_v2(D,X,M,L,C) + # interface runs a checkpoint operation on database X of database + # connection D in mode M. Status information is written back into + # integers pointed to by L and C. + # + # Tests 1, 2 and 3 below verify the "on database X" part of the + # above. Other parts of this requirement are tested below. + # + # EVIDENCE-OF: R-00653-06026 If parameter zDb is NULL or points to a + # zero length string, then the specified operation is attempted on all + # WAL databases attached to database connection db. + # + # Tests 4 and 5 below test this. + # + foreach {tn2 zDb dblist} { + 1 main test.db + 2 aux test.db2 + 3 aux2 test.db3 + 4 "" {test.db test.db2 test.db3} + 5 - {test.db test.db2 test.db3} + 6 temp {} + } { + do_test $tn.1.$tn2 { + execsql { + INSERT INTO t1 VALUES(1); + INSERT INTO t2 VALUES(2); + INSERT INTO t3 VALUES(3); + } + save_db_hashes + + if {$zDb == "-"} { + checkpoint db passive + } else { + checkpoint db passive $zDb + } + + compare_db_hashes + } $dblist + } + + # EVIDENCE-OF: R-38207-48996 If zDb is not NULL (or a zero length + # string) and is not the name of any attached database, SQLITE_ERROR is + # returned to the caller. + do_test $tn.2.1 { + list [catch { checkpoint db passive notadb } msg] $msg + } {1 {SQLITE_ERROR - unknown database: notadb}} + + # EVIDENCE-OF: R-14303-42483 If database zDb is the name of an attached + # database that is not in WAL mode, SQLITE_OK is returned and both + # *pnLog and *pnCkpt set to -1. + # + if {$tn==3} { + # With sqlite3_wal_checkpoint() the two output variables cannot be + # tested. So just test that no error is returned when attempting to + # checkpoint a db in rollback mode. + do_test $tn.2.2.a { checkpoint db passive aux3 } {} + } else { + do_test $tn.2.2.b { checkpoint db passive aux3 } {0 -1 -1} + } + + # EVIDENCE-OF: R-62028-47212 All calls obtain an exclusive "checkpoint" + # lock on the database file. + db close + testvfs tvfs + tvfs filter xShmLock + tvfs script filelock + proc filelock {method file handle details} { + # Test for an exclusive checkpoint lock. A checkpoint lock locks a + # single byte starting at offset 1. + if {$details == "1 1 lock exclusive"} { set ::seen_checkpoint_lock 1 } + } + sqlite3 db test.db -vfs tvfs + do_test $tn.3.1 { + execsql { INSERT INTO t1 VALUES('xyz') } + unset -nocomplain ::seen_checkpoint_lock + checkpoint db passive + set ::seen_checkpoint_lock + } {1} + db close + tvfs delete + reset_db + + + + + #----------------------------------------------------------------------- + # EVIDENCE-OF: R-10421-19736 If any other process is running a + # checkpoint operation at the same time, the lock cannot be obtained and + # SQLITE_BUSY is returned. + # + # EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + # it will not be invoked in this case. + # + testvfs tvfs + tvfs filter xWrite + sqlite3 db test.db -vfs tvfs + sqlite3 db2 test.db -vfs tvfs + + do_test $tn.3.2.1 { + db2 eval { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1,2); + INSERT INTO t1 VALUES(3,4); + INSERT INTO t1 VALUES(5,6); + } + file size test.db-wal + } [wal_file_size 5 1024] + + + # Connection [db] runs a checkpoint. During this checkpoint, each + # time it calls xWrite() to write a page into the database file, we + # attempt to start a checkpoint using [db2]. According to the + # first requirement being tested, this should return SQLITE_BUSY. According + # to the second, the busy-handler belonging to [db2] should not be + # invoked. + # + set ::write_count 0 + set ::write_errors [list] + proc busy_callback {args} { + lappend ::write_errors "busy handler called!" + } + proc write_callback {args} { + set rc [catch {checkpoint db2 passive} msg] + if {0==[regexp "database is locked" $msg] && $msg!="1 -1 -1"} { + lappend ::write_errors "$rc $msg" + } + incr ::write_count + } + db2 busy busy_callback + tvfs script write_callback + + do_test $tn.3.2.2 { + db eval {SELECT * FROM sqlite_master} + checkpoint db full + set ::write_count + } {2} + + do_test $tn.3.2.3 { + set ::write_errors + } {} + + db close + db2 close + tvfs delete + +} + +#----------------------------------------------------------------------- +# EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint +# mode: +# +# Valid checkpoint modes are 0, 1, 2 and 3. +# +sqlite3 db test.db +foreach {tn mode res} { + 0 -1001 {1 {SQLITE_MISUSE - not an error}} + 1 -1 {1 {SQLITE_MISUSE - not an error}} + 2 0 {0 {0 -1 -1}} + 3 1 {0 {0 -1 -1}} + 4 2 {0 {0 -1 -1}} + 5 3 {0 {0 -1 -1}} + 6 4 {1 {SQLITE_MISUSE - not an error}} + 7 114 {1 {SQLITE_MISUSE - not an error}} + 8 1000000 {1 {SQLITE_MISUSE - not an error}} +} { + do_test 4.$tn { + list [catch "sqlite3_wal_checkpoint_v2 db $mode" msg] $msg + } $res +} + + +finish_test diff --git a/test/join5.test b/test/join5.test index 45d8a31596..b0b0df4f9d 100644 --- a/test/join5.test +++ b/test/join5.test @@ -106,5 +106,59 @@ do_test join5-2.12 { execsql {SELECT * FROM xy LEFT JOIN ab ON NULL WHERE NULL} } {} +# Ticket https://www.sqlite.org/src/tktview/6f2222d550f5b0ee7ed37601 +# Incorrect output on a LEFT JOIN. +# +do_execsql_test join5-3.1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + DROP TABLE IF EXISTS t3; + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(1); + CREATE TABLE x2(b NOT NULL); + CREATE TABLE x3(c, d); + INSERT INTO x3 VALUES('a', NULL); + INSERT INTO x3 VALUES('b', NULL); + INSERT INTO x3 VALUES('c', NULL); + SELECT * FROM x1 LEFT JOIN x2 LEFT JOIN x3 ON x3.d = x2.b; +} {1 {} {} {}} +do_execsql_test join5-3.2 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + DROP TABLE IF EXISTS t3; + DROP TABLE IF EXISTS t4; + DROP TABLE IF EXISTS t5; + CREATE TABLE t1(x text NOT NULL, y text); + CREATE TABLE t2(u text NOT NULL, x text NOT NULL); + CREATE TABLE t3(w text NOT NULL, v text); + CREATE TABLE t4(w text NOT NULL, z text NOT NULL); + CREATE TABLE t5(z text NOT NULL, m text); + INSERT INTO t1 VALUES('f6d7661f-4efe-4c90-87b5-858e61cd178b',NULL); + INSERT INTO t1 VALUES('f6ea82c3-2cad-45ce-ae8f-3ddca4fb2f48',NULL); + INSERT INTO t1 VALUES('f6f47499-ecb4-474b-9a02-35be73c235e5',NULL); + INSERT INTO t1 VALUES('56f47499-ecb4-474b-9a02-35be73c235e5',NULL); + INSERT INTO t3 VALUES('007f2033-cb20-494c-b135-a1e4eb66130c', + 'f6d7661f-4efe-4c90-87b5-858e61cd178b'); + SELECT * + FROM t3 + INNER JOIN t1 ON t1.x= t3.v AND t1.y IS NULL + LEFT JOIN t4 ON t4.w = t3.w + LEFT JOIN t5 ON t5.z = t4.z + LEFT JOIN t2 ON t2.u = t5.m + LEFT JOIN t1 xyz ON xyz.y = t2.x; +} {007f2033-cb20-494c-b135-a1e4eb66130c f6d7661f-4efe-4c90-87b5-858e61cd178b f6d7661f-4efe-4c90-87b5-858e61cd178b {} {} {} {} {} {} {} {} {}} +do_execsql_test join5-3.3 { + DROP TABLE IF EXISTS x1; + DROP TABLE IF EXISTS x2; + DROP TABLE IF EXISTS x3; + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(1); + CREATE TABLE x2(b NOT NULL); + CREATE TABLE x3(c, d); + INSERT INTO x3 VALUES('a', NULL); + INSERT INTO x3 VALUES('b', NULL); + INSERT INTO x3 VALUES('c', NULL); + SELECT * FROM x1 LEFT JOIN x2 JOIN x3 WHERE x3.d = x2.b; +} {} finish_test diff --git a/test/permutations.test b/test/permutations.test index b3c9e58934..20e253273e 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -113,6 +113,7 @@ set allquicktests [test_set $alltests -exclude { vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test sort3.test sort4.test fts4growth.test fts4growth2.test + bigsort.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] diff --git a/test/scanstatus.test b/test/scanstatus.test index 7713bae5fc..ed24d97437 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -268,7 +268,7 @@ do_scanstatus_test 4.2.2 { nLoop 1 nVisit 1 nEst 1.0 zName sqlite_autoindex_p1_1 zExplain {SEARCH TABLE p1 USING INDEX sqlite_autoindex_p1_1 (x=?)} - nLoop 1 nVisit 3 nEst 524288.0 zName c1 zExplain {SCAN TABLE c1} + nLoop 1 nVisit 3 nEst 262144.0 zName c1 zExplain {SCAN TABLE c1} } #------------------------------------------------------------------------- diff --git a/test/sharedB.test b/test/sharedB.test new file mode 100644 index 0000000000..1b15755076 --- /dev/null +++ b/test/sharedB.test @@ -0,0 +1,60 @@ +# 2014-12-05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Open two database connections on the same database in shared cache +# mode. Hold one open while repeatedly closing, reopening, and using +# the second. +# +# This test is designed to demostrate that the fix for ticket +# [e4a18565a36884b00edf66541f38c693827968ab] works. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +if {[run_thread_tests]==0} { finish_test ; return } +db close +set ::testprefix sharedB + +set ::enable_shared_cache [sqlite3_enable_shared_cache 1] + +#------------------------------------------------------------------------- +# +do_test 1.1 { + sqlite3 db1 test.db + sqlite3 db2 test.db + + db1 eval { + CREATE TABLE t1(x,y TEXT COLLATE nocase); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<100) + INSERT INTO t1(x,y) SELECT i, printf('x%03dy',i) FROM c; + CREATE INDEX t1yx ON t1(y,x); + } + db2 eval { + SELECT x FROM t1 WHERE y='X014Y'; + } +} {14} + +for {set j 1} {$j<=100} {incr j} { + do_test 1.2.$j { + db2 close + sqlite3 db2 test.db + db2 eval { + SELECT x FROM t1 WHERE y='X014Y'; + } + } {14} +} + +db1 close +db2 close +sqlite3_enable_shared_cache $::enable_shared_cache +finish_test diff --git a/test/shell1.test b/test/shell1.test index ab382e74cb..27bbe87259 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -45,20 +45,21 @@ do_test shell1-1.1.1 { list $rc \ [regexp {Error: unknown option: -bad} $res] } {1 1} +do_test shell1-1.1.1b { + set res [catchcmd "test.db -bad" ""] + set rc [lindex $res 0] + list $rc \ + [regexp {Error: unknown option: -bad} $res] +} {1 1} # error on extra options do_test shell1-1.1.2 { - set res [catchcmd "-bad test.db \"select 3\" \"select 4\"" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "select 4"} $res] -} {1 1} + catchcmd "test.db \"select 3\" \"select 4\"" "" +} {0 {3 +4}} # error on extra options do_test shell1-1.1.3 { - set res [catchcmd "-bad FOO test.db BAD" ".quit"] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "BAD"} $res] -} {1 1} + catchcmd "test.db FOO test.db BAD" ".quit" +} {1 {Error: near "FOO": syntax error}} # -help do_test shell1-1.2.1 { @@ -75,11 +76,11 @@ do_test shell1-1.3.1 { catchcmd "-init FOO test.db" "" } {0 {}} do_test shell1-1.3.2 { - set res [catchcmd "-init FOO test.db .quit BAD" ""] - set rc [lindex $res 0] - list $rc \ - [regexp {Error: too many options: "BAD"} $res] -} {1 1} + catchcmd "-init FOO test.db .quit BAD" "" +} {0 {}} +do_test shell1-1.3.3 { + catchcmd "-init FOO test.db BAD .quit" "" +} {1 {Error: near "BAD": syntax error}} # -echo print commands before execution do_test shell1-1.4.1 { diff --git a/test/shell2.test b/test/shell2.test index def574c76c..616610bd4b 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -52,9 +52,9 @@ do_test shell2-1.1.1 { # Ticket [f5cb008a65]. do_test shell2-1.2.1 { set rc [catch { eval exec $CLI \":memory:\" \"select 3\" \"select 4\" } msg] - list $rc \ - [regexp {Error: too many options: "select 4"} $msg] -} {1 1} + list $rc $msg +} {0 {3 +4}} # Test a problem reported on the mailing list. The shell was at one point # returning the generic SQLITE_ERROR message ("SQL error or missing database") diff --git a/test/vtab1.test b/test/vtab1.test index 0542ee6fdd..2929b1e54d 100644 --- a/test/vtab1.test +++ b/test/vtab1.test @@ -1395,4 +1395,46 @@ do_execsql_test 21.3 { SELECT * FROM t9v WHERE a=b; } {2 2 2} +#------------------------------------------------------------------------- +# At one point executing a CREATE VIRTUAL TABLE statement that specified +# a database name but no virtual table arguments was causing an internal +# buffer overread. Valgrind would report errors while running the following +# tests. Specifically: +# +# CREATE VIRTUAL TABLE t1 USING fts4; -- Ok - no db name. +# CREATE VIRTUAL TABLE main.t1 USING fts4(x); -- Ok - has vtab arguments. +# CREATE VIRTUAL TABLE main.t1 USING fts4; -- Had the problem. +# +ifcapable fts3 { + forcedelete test.db2 + set nm [string repeat abcdefghij 100] + do_execsql_test 22.1 { + ATTACH 'test.db2' AS $nm + } + + execsql "SELECT * FROM sqlite_master" + do_execsql_test 22.2 "CREATE VIRTUAL TABLE ${nm}.t1 USING fts4" + + do_test 22.3.1 { + set sql "CREATE VIRTUAL TABLE ${nm}.t2 USING fts4" + set stmt [sqlite3_prepare_v2 db $sql -1 dummy] + sqlite3_step $stmt + } {SQLITE_DONE} + + do_test 22.3.2 { + sqlite3_finalize $stmt + } {SQLITE_OK} + + do_test 22.4.1 { + set sql "CREATE VIRTUAL TABLE ${nm}.t3 USING fts4" + set n [string length $sql] + set stmt [sqlite3_prepare db "${sql}xyz" $n dummy] + sqlite3_step $stmt + } {SQLITE_DONE} + + do_test 22.4.2 { + sqlite3_finalize $stmt + } {SQLITE_OK} +} + finish_test diff --git a/test/wal5.test b/test/wal5.test index 68750f1479..09c7d76040 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -55,7 +55,8 @@ foreach {testprefix do_wal_checkpoint} { if {[lsearch {-mode -db} $key]<0} { error "unknown switch: $key" } } - if {$a(-mode)!="restart" && $a(-mode)!="full"} { set a(-mode) passive } + set vals {restart full truncate} + if {[lsearch -exact $vals $a(-mode)]<0} { set a(-mode) passive } set cmd [list sqlite3_wal_checkpoint_v2 $dbhandle $a(-mode)] if {[info exists a(-db)]} { lappend sql $a(-db) } @@ -278,6 +279,11 @@ foreach {testprefix do_wal_checkpoint} { 9 RESTART 2 {1 4 3} 2 10 RESTART 3 {1 4 4} 3 + 11 TRUNCATE - {0 0 0} 3 + 12 TRUNCATE 1 {1 3 3} 1 + 13 TRUNCATE 2 {1 4 3} 2 + 14 TRUNCATE 3 {1 4 4} 3 + } { do_multiclient_test tn { setup_and_attach_aux @@ -348,6 +354,41 @@ foreach {testprefix do_wal_checkpoint} { do_test 3.$tn.6 { code3 { do_wal_checkpoint db3 } } {0 0 0} } + + # Test SQLITE_CHECKPOINT_TRUNCATE. + # + do_multiclient_test tn { + + code1 $do_wal_checkpoint + code2 $do_wal_checkpoint + code3 $do_wal_checkpoint + + do_test 3.$tn.1 { + sql1 { + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = normal; + CREATE TABLE t1(x, y); + CREATE INDEX i1 ON t1(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } + file size test.db-wal + } [wal_file_size 8 1024] + + do_test 3.$tn.2 { do_wal_checkpoint db -mode truncate } {0 0 0} + do_test 3.$tn.3 { file size test.db-wal } 0 + + do_test 3.$tn.4 { + sql2 { SELECT * FROM t1 } + } {1 2 3 4} + + do_test 3.$tn.5 { + sql2 { INSERT INTO t1 VALUES('a', 'b') } + file size test.db-wal + } [wal_file_size 2 1024] + + } } diff --git a/test/whereJ.test b/test/whereJ.test index 8431c3a4b0..48924d0fcf 100644 --- a/test/whereJ.test +++ b/test/whereJ.test @@ -640,4 +640,39 @@ do_execsql_test 4.2 { } {/.*SCAN TABLE cx.*SEARCH TABLE px.*SEARCH TABLE le.*/} +# The following test is derived from a performance problem reported from +# the field. Notice the multiple indexes with the same initial tables, +# and the unusual WHERE clause terms. +# +do_test 5.1 { + set res [db eval { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a,b,c,d,e,f,g,h); + CREATE INDEX t1abc ON t1(a,b,c); + CREATE INDEX t1abe ON t1(a,b,e); + CREATE INDEX t1abf ON t1(a,b,f); + ANALYZE; + DROP TABLE IF EXISTS sqlite_stat4; + DROP TABLE IF EXISTS sqlite_stat3; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1abc','2000000 8000 1600 800'), + ('t1','t1abe','2000000 8000 1600 150'), + ('t1','t1abf','2000000 8000 1600 150'); + ANALYZE sqlite_master; + + EXPLAIN QUERY PLAN + SELECT * FROM t1 + WHERE (a=1 OR a=2) + AND (b=3 OR b=4) + AND (d>=5 AND d<=5) + AND ((e>=7 AND e<=7) OR (f>=8 AND f<=8)) + AND g>0; + }] +} {~/ANY/} +do_test 5.2 {set res} {/USING INDEX t1abe/} +do_test 5.3 {set res} {/USING INDEX t1abf/} + + + finish_test