From 3eae6664a00961baf3d011229794cd98371aa9cf Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 20 Jan 2024 16:18:04 +0000 Subject: [PATCH 1/8] Allow "_" characters to appear following any digit in an integer or real SQL literal. FossilOrigin-Name: 401650aaccbc99246bd4e1ff37a28b78f528178aee2f294d87b9f7fecd7432bb --- manifest | 29 +++++++++++++++------------- manifest.uuid | 2 +- src/parse.y | 6 ++++++ src/sqliteInt.h | 3 +++ src/tokenize.c | 27 +++++++++++++++++++------- src/util.c | 21 +++++++++++++++++++++ test/literal.test | 3 +++ test/speedtest1.c | 46 +++++++++++++++++++++++++++++++++++++++++++++ tool/speed-check.sh | 3 +++ 9 files changed, 119 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 320c5bbcb4..f2673bb861 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sdoing\sa\stext-affinity\scomparison\sbetween\stwo\svalues\swhere\sone\sor\sboth\nhave\sboth\sa\stext\sand\sa\snumeric\stype,\smake\ssure\sthe\snumeric\stype\sdoes\snot\nconfuse\sthe\sanswer.\s\sThis\sis\sa\sdeeper\sfix\sto\sthe\sproblem\sobserved\sby\n[forum:/forumpost/3776b48e71|forum\spose\s3776b48e71].\s\sThe\sproblem\sbisects\nto\s[25f2246be404f38b]\son\s2014-08-24,\sprior\sto\sversion\s3.8.7. -D 2024-01-20T15:13:13.896 +C Allow\s"_"\scharacters\sto\sappear\sfollowing\sany\sdigit\sin\san\sinteger\sor\sreal\sSQL\sliteral. +D 2024-01-20T16:18:04.854 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -727,7 +727,7 @@ F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a -F src/parse.y 020d80386eb216ec9520549106353c517d2bbc89be28752ffdca649a9eaf56ec +F src/parse.y 2354aaf964e7c4154a9dbe56ea55a797a0fa3021c38b50afe491ea4a387bf971 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 @@ -743,7 +743,7 @@ F src/shell.c.in d1ed426aae2d547932971e8019939cacb4dfda8258e45b8924b250e488e2d53 F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 73800d73e21180e6b3df8d0fe7d11758dc24367fd2b0b0075b48fc116de406bb +F src/sqliteInt.h 6e5e330d84b4ace70e3163721601f01df84566e6db21e1fc45bd00636e3d6640 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -800,13 +800,13 @@ F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb7 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 23d9f4539880b40226254ad9072f4ecf12eb1902e62aea47aac29928afafcfd5 +F src/tokenize.c b3ab5ee644f3ac54e6a6a856cfb78bdb285e634fae39b38ce0f6e848305ca32e F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70ee F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e -F src/util.c 078f040366d5bd5f47658d045f901c768c1c636c6eaea121f3a1cbd63c3edb5b +F src/util.c 2b274b03f0d4fd1bf253f160d6b399bc5265338b043c078045c5e32298891ec8 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 92910d536e0b77505599cd6ae5d9d449e4a5d31ada61da4c0bb84f6ccb2c3189 F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb @@ -1355,7 +1355,7 @@ F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3 F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de98745c F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e -F test/literal.test 31ef70d425c39b993d0dd61b3f4c65d30b7491e8b3dbbd2d8a23b78b1e8931ca +F test/literal.test 9bf0ce5ca1034e3c6f83d5dff82abdedcfa1596cd843b2e4f93f58828e123046 F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db @@ -1624,7 +1624,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c -F test/speedtest1.c f9505ad6f9c2c3c488a370a2d193e9603a030e51126ef3ecfeb056d21f0e7ad5 +F test/speedtest1.c 19c9b60908d25502d2831f97efee8b81006c356ab8c08327e25d24a4144f2131 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -2112,7 +2112,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c F tool/showwal.c 11eca547980a066b081f512636151233350ac679f29ecf4ebfce7f4530230b3d F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl 70c87c04cfd2e77b3e6f21c33ca768296aa8f67d4ab4874786ac8fbb28433477 -F tool/speed-check.sh 72dc85b2c0484af971ee3e7d10775f72b4e771e27e162c2099b3bf25517c25fb +F tool/speed-check.sh e8d20cc2eb9c85ec1ba562226de144435456dcdff4ee618de49603c6958f6116 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -2159,8 +2159,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 01868ebcd25fadb2034da234c0636e82d07c5abc902ef66493cadfc988e74d7b -R c017e3ff6a3cf5fef0c29004a344e6f7 -U drh -Z f4a73b0af78f34dd85f781f1eb7187c3 +P 709841f88c77276f09701bf38e25503c64b3a0afbe2fbf878136db12f31cbe21 +R 25ab10e7bacd9206b22870bd4ed39a9a +T *branch * digit-separators +T *sym-digit-separators * +T -sym-trunk * +U dan +Z ce3fc1dbdae9b70fcb4dc10489ba2fab # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d23f19cb29..e6ba736181 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -709841f88c77276f09701bf38e25503c64b3a0afbe2fbf878136db12f31cbe21 \ No newline at end of file +401650aaccbc99246bd4e1ff37a28b78f528178aee2f294d87b9f7fecd7432bb \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 19491192e3..b0d03a04ee 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1923,6 +1923,12 @@ filter_clause(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } SPAN /* The span operator */ ERROR /* An expression containing an error */ . + +term(A) ::= QNUMBER(X). { + A=tokenExpr(pParse,@X,X); + sqlite3DequoteNumber(A); +} + /* There must be no more than 255 tokens defined above. If this grammar ** is extended with new rules and tokens, they must either be so few in ** number that TK_SPAN is no more than 255, or else the new tokens must diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 79a36e060e..c4aaf9d203 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -609,6 +609,8 @@ # define SQLITE_OMIT_ALTERTABLE #endif +#define SQLITE_DIGIT_SEPARATOR '_' + /* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() @@ -4792,6 +4794,7 @@ int sqlite3ErrorToParser(sqlite3*,int); void sqlite3Dequote(char*); void sqlite3DequoteExpr(Expr*); void sqlite3DequoteToken(Token*); +void sqlite3DequoteNumber(Expr*); void sqlite3TokenInit(Token*,char*); int sqlite3KeywordCode(const unsigned char*, int); int sqlite3RunParser(Parse*, const char*); diff --git a/src/tokenize.c b/src/tokenize.c index f4d013deea..ae1f6da3f1 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -441,21 +441,34 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ return i; } #endif - for(i=0; sqlite3Isdigit(z[i]); i++){} + for(i=0; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ *tokenType = TK_QNUMBER; } + else{ break; } + } + } #ifndef SQLITE_OMIT_FLOATING_POINT if( z[i]=='.' ){ - i++; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i++; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ *tokenType = TK_QNUMBER; } + else{ break; } + } + } } if( (z[i]=='e' || z[i]=='E') && ( sqlite3Isdigit(z[i+1]) || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) ) ){ - i += 2; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i+=2; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ *tokenType = TK_QNUMBER; } + else{ break; } + } + } } #endif while( IdChar(z[i]) ){ diff --git a/src/util.c b/src/util.c index 207b901bad..24eff14e9f 100644 --- a/src/util.c +++ b/src/util.c @@ -311,6 +311,27 @@ void sqlite3DequoteExpr(Expr *p){ sqlite3Dequote(p->u.zToken); } +/* +** Expression p is a QINTEGER or QFLOAT (quoted integer or float). Dequote +** the value in p->u.zToken and set the type to INTEGER or FLOAT. "Quoted" +** integers or floats are those that contain '_' characters that must +** be removed before further processing. +*/ +void sqlite3DequoteNumber(Expr *p){ + if( p ){ + const char *pIn = p->u.zToken; + char *pOut = p->u.zToken; + assert( p->op==TK_QNUMBER ); + p->op = TK_INTEGER; + do { + if( *pIn!=SQLITE_DIGIT_SEPARATOR ){ + *pOut++ = *pIn; + if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT; + } + }while( *pIn++ ); + } +} + /* ** If the input token p is quoted, try to adjust the token to remove ** the quotes. This is not always possible: diff --git a/test/literal.test b/test/literal.test index 7bbb8f0f7a..8537ab5935 100644 --- a/test/literal.test +++ b/test/literal.test @@ -44,4 +44,7 @@ test_literal 1.12 '-0xFF' text -0xFF test_literal 1.13 -'0xFF' integer 0 +test_literal 2.1 1_000 integer 1000 + + finish_test diff --git a/test/speedtest1.c b/test/speedtest1.c index 4f32580987..5709423818 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -2150,6 +2150,50 @@ void testset_debug1(void){ } } +/* +** This testset focuses on the speed of parsing numeric literals (integers +** and real numbers). This was added to test the impact of allowing "_" +** characters to appear in numeric SQL literals to make them easier to read. +** For example, "SELECT 1_000_000;" instead of "SELECT 1000000;". +*/ +void testset_parsenumber(void){ + const char *zSql1 = "SELECT 1, 12, 123, 1234, 12345, 123456"; + const char *zSql2 = "SELECT 8227256643844975616, 7932208612563860480, " + "2010730661871032832, 9138463067404021760, " + "2557616153664746496, 2557616153664746496"; + const char *zSql3 = "SELECT 1.0, 1.2, 1.23, 123.4, 1.2345, 1.23456"; + const char *zSql4 = "SELECT 8.227256643844975616, 7.932208612563860480, " + "2.010730661871032832, 9.138463067404021760, " + "2.557616153664746496, 2.557616153664746496"; + + const int NROW = 100*g.szTest; + int ii; + + speedtest1_begin_test(100, "parsing small integers"); + for(ii=0; ii #include @@ -2557,6 +2601,8 @@ int main(int argc, char **argv){ testset_fp(); }else if( strcmp(zThisTest,"trigger")==0 ){ testset_trigger(); + }else if( strcmp(zThisTest,"parsenumber")==0 ){ + testset_parsenumber(); }else if( strcmp(zThisTest,"rtree")==0 ){ #ifdef SQLITE_ENABLE_RTREE testset_rtree(6, 147); diff --git a/tool/speed-check.sh b/tool/speed-check.sh index 4cc25794a2..5d425c3b37 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -161,6 +161,9 @@ while test "$1" != ""; do --fp) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp" ;; + --parsenumber) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset parsenumber" + ;; --stmtscanstatus) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtscanstatus" ;; From 95295a7e9b59e90029ae7a90ebee1667c51a3139 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 20 Jan 2024 16:46:25 +0000 Subject: [PATCH 2/8] Add test cases for the new code on this branch. FossilOrigin-Name: 49f29a7b4f44f691ecf7a57b26477971e6ffaf2a0b1690b4926ffbb43fa3929b --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/tokenize.c | 7 +++++-- test/literal.test | 11 +++++++++-- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f2673bb861..85784dcc72 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\s"_"\scharacters\sto\sappear\sfollowing\sany\sdigit\sin\san\sinteger\sor\sreal\sSQL\sliteral. -D 2024-01-20T16:18:04.854 +C Add\stest\scases\sfor\sthe\snew\scode\son\sthis\sbranch. +D 2024-01-20T16:46:25.186 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -800,7 +800,7 @@ F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb7 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c b3ab5ee644f3ac54e6a6a856cfb78bdb285e634fae39b38ce0f6e848305ca32e +F src/tokenize.c e0cdd971ebf7225cd617661d8198fc90a49559e21210af03bdf440e2fcba6b88 F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70ee F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 @@ -1355,7 +1355,7 @@ F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3 F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de98745c F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e -F test/literal.test 9bf0ce5ca1034e3c6f83d5dff82abdedcfa1596cd843b2e4f93f58828e123046 +F test/literal.test 62415688179c0d586f3633482e6c8e441edb805cd73c306964cf39912a6bff0e F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db @@ -2159,11 +2159,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 709841f88c77276f09701bf38e25503c64b3a0afbe2fbf878136db12f31cbe21 -R 25ab10e7bacd9206b22870bd4ed39a9a -T *branch * digit-separators -T *sym-digit-separators * -T -sym-trunk * +P 401650aaccbc99246bd4e1ff37a28b78f528178aee2f294d87b9f7fecd7432bb +R f613ae2f01e87601e8fa7113cad6d125 U dan -Z ce3fc1dbdae9b70fcb4dc10489ba2fab +Z 0deb0f7f0bb54434edd39b41a13f24b4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e6ba736181..a510d65faa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -401650aaccbc99246bd4e1ff37a28b78f528178aee2f294d87b9f7fecd7432bb \ No newline at end of file +49f29a7b4f44f691ecf7a57b26477971e6ffaf2a0b1690b4926ffbb43fa3929b \ No newline at end of file diff --git a/src/tokenize.c b/src/tokenize.c index ae1f6da3f1..592d6b8d8d 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -635,10 +635,13 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ if( tokenType>=TK_WINDOW ){ assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + || tokenType==TK_QNUMBER ); #else if( tokenType>=TK_SPACE ){ - assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL + || tokenType==TK_QNUMBER + ); #endif /* SQLITE_OMIT_WINDOWFUNC */ if( AtomicLoad(&db->u1.isInterrupted) ){ pParse->rc = SQLITE_INTERRUPT; @@ -671,7 +674,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ assert( n==6 ); tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); #endif /* SQLITE_OMIT_WINDOWFUNC */ - }else{ + }else if( tokenType!=TK_QNUMBER ){ Token x; x.z = zSql; x.n = n; diff --git a/test/literal.test b/test/literal.test index 8537ab5935..01ea7b954c 100644 --- a/test/literal.test +++ b/test/literal.test @@ -43,8 +43,15 @@ test_literal 1.11 '0xFF' text 0xFF test_literal 1.12 '-0xFF' text -0xFF test_literal 1.13 -'0xFF' integer 0 +test_literal 2.1 1_000 integer 1000 +test_literal 2.2 1.1_1 real 1.11 +test_literal 2.3 1_0.1_1 real 10.11 +test_literal 2.4 1e1_000 real Inf +test_literal 2.5 123______456.7_8__9_ real 123456.789 +test_literal 2.6 9_223_372_036_854_775_807 integer 9223372036854775807 +test_literal 2.7 9_223_372_036_854_775_808 real 9.22337203685478e+18 +test_literal 2.8 -9_223_372_036_854_775_808 integer -9223372036854775808 -test_literal 2.1 1_000 integer 1000 - +test_literal 3.3 1e12 real 1000000000000.0 finish_test From 17589f781142afbe9ccb00fb6912270a50e2f710 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 22 Jan 2024 11:12:38 +0000 Subject: [PATCH 3/8] Fix test script literal.test so that it works with SQLITE_OMIT_ALTER_TABLE builds. FossilOrigin-Name: b57327be4be9638f5ee834a6c9e06135aaef9e499516f8b53f927ffe81033f61 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/literal.test | 19 ++++++++++++++----- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 6c05005d18..d5565a35b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2024-01-20T18:45:30.244 +C Fix\stest\sscript\sliteral.test\sso\sthat\sit\sworks\swith\sSQLITE_OMIT_ALTER_TABLE\sbuilds. +D 2024-01-22T11:12:38.218 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1355,7 +1355,7 @@ F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3 F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de98745c F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e -F test/literal.test 17d21dda2f877018a02c28e2a85947b82ed638f2ca5ef3fa47c92359f02520a3 +F test/literal.test db71a7616dc027badf8d8f7ac6e23e8655915eb51b187c40a0a6e667aed495b7 F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db @@ -2159,8 +2159,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 49f29a7b4f44f691ecf7a57b26477971e6ffaf2a0b1690b4926ffbb43fa3929b 298d6977285c71be917896bc875a8a26d985dcf0a74069b7c4d290e8ff0ac618 -R d4f77c6633329aa6323323954676adb8 +P 03ade4a810516ff84bc5c1a716b0e454cc8a89374100415ff21a82449112811b +R 4dc96dbb63712d691b87fb65e2e995c9 U dan -Z 02dbf9257e243f2009f448d2d8147299 +Z 060bde75456debf514e23a793c3f90b5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5f5c21e2d7..902662ca4b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03ade4a810516ff84bc5c1a716b0e454cc8a89374100415ff21a82449112811b \ No newline at end of file +b57327be4be9638f5ee834a6c9e06135aaef9e499516f8b53f927ffe81033f61 \ No newline at end of file diff --git a/test/literal.test b/test/literal.test index e29789d37d..3140eedf2a 100644 --- a/test/literal.test +++ b/test/literal.test @@ -19,12 +19,21 @@ set ::testprefix literal proc test_literal {tn lit type val} { do_execsql_test $tn.1 "SELECT typeof( $lit ), $lit" [list $type $val] - do_execsql_test $tn.2 " + ifcapable altertable { + do_execsql_test $tn.2 " + DROP TABLE IF EXISTS x1; + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(123); + ALTER TABLE x1 ADD COLUMN b DEFAULT $lit ; + SELECT typeof(b), b FROM x1; + " [list $type $val] + } + + do_execsql_test $tn.3 " DROP TABLE IF EXISTS x1; - CREATE TABLE x1(a); - INSERT INTO x1 VALUES(123); - ALTER TABLE x1 ADD COLUMN b DEFAULT $lit ; - SELECT typeof(b), b FROM x1; + CREATE TABLE x1(a DEFAULT $lit); + INSERT INTO x1 DEFAULT VALUES; + SELECT typeof(a), a FROM x1; " [list $type $val] } From 87ad06ed997a505b60d2ca9c9751c0d0e83cfbd4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 22 Jan 2024 17:18:41 +0000 Subject: [PATCH 4/8] Do not allow digit separators that are not surrounded on both sides by digits. FossilOrigin-Name: 5bbc378a3022e2013048c8e296d95de21d15a8d7c1b62e8ffb64782f952cfe12 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/tokenize.c | 20 ++++++++++++++++---- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index d5565a35b7..b43db80a8c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stest\sscript\sliteral.test\sso\sthat\sit\sworks\swith\sSQLITE_OMIT_ALTER_TABLE\sbuilds. -D 2024-01-22T11:12:38.218 +C Do\snot\sallow\sdigit\sseparators\sthat\sare\snot\ssurrounded\son\sboth\ssides\sby\sdigits. +D 2024-01-22T17:18:41.963 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -800,7 +800,7 @@ F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb7 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c e0cdd971ebf7225cd617661d8198fc90a49559e21210af03bdf440e2fcba6b88 +F src/tokenize.c 1664cd0d9224bf63e7471d892074f580933c0e85a3b77f98e016dce2d5fed4a5 F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70ee F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 @@ -2159,8 +2159,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 03ade4a810516ff84bc5c1a716b0e454cc8a89374100415ff21a82449112811b -R 4dc96dbb63712d691b87fb65e2e995c9 -U dan -Z 060bde75456debf514e23a793c3f90b5 +P b57327be4be9638f5ee834a6c9e06135aaef9e499516f8b53f927ffe81033f61 +R dc78fe3596a95d4e421310d3dc095c60 +U drh +Z 8ae254187429ecb3341b495c0d5f92df # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 902662ca4b..40427080bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b57327be4be9638f5ee834a6c9e06135aaef9e499516f8b53f927ffe81033f61 \ No newline at end of file +5bbc378a3022e2013048c8e296d95de21d15a8d7c1b62e8ffb64782f952cfe12 \ No newline at end of file diff --git a/src/tokenize.c b/src/tokenize.c index 592d6b8d8d..37254da70d 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -443,8 +443,14 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ #endif for(i=0; 1; i++){ if( sqlite3Isdigit(z[i])==0 ){ - if( z[i]==SQLITE_DIGIT_SEPARATOR ){ *tokenType = TK_QNUMBER; } - else{ break; } + if( z[i]==SQLITE_DIGIT_SEPARATOR + && sqlite3Isdigit(z[i-1]) + && sqlite3Isdigit(z[i+1]) + ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } } } #ifndef SQLITE_OMIT_FLOATING_POINT @@ -452,8 +458,14 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; for(i++; 1; i++){ if( sqlite3Isdigit(z[i])==0 ){ - if( z[i]==SQLITE_DIGIT_SEPARATOR ){ *tokenType = TK_QNUMBER; } - else{ break; } + if( z[i]==SQLITE_DIGIT_SEPARATOR + && sqlite3Isdigit(z[i-1]) + && sqlite3Isdigit(z[i+1]) + ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } } } } From fbb72fae8ff79bdc1308f70c64a8093c9d605ce8 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 22 Jan 2024 19:00:50 +0000 Subject: [PATCH 5/8] Add extra tests for the code on this branch. FossilOrigin-Name: b15bbb201a7727806b9fd6c02f2579b01e0e38155a8f9e24c5d6f487fe3e8b6c --- manifest | 18 +++++++++-------- manifest.uuid | 2 +- src/tokenize.c | 10 ++++++++-- test/literal.test | 28 ++++++++++++++++++++++++++- test/literal2.tcl | 27 ++++++++++++++++++++++++++ test/literal2.test | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 test/literal2.tcl create mode 100644 test/literal2.test diff --git a/manifest b/manifest index b43db80a8c..c1b974d23a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sdigit\sseparators\sthat\sare\snot\ssurrounded\son\sboth\ssides\sby\sdigits. -D 2024-01-22T17:18:41.963 +C Add\sextra\stests\sfor\sthe\scode\son\sthis\sbranch. +D 2024-01-22T19:00:50.821 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -800,7 +800,7 @@ F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb7 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 1664cd0d9224bf63e7471d892074f580933c0e85a3b77f98e016dce2d5fed4a5 +F src/tokenize.c f283d8017b57d8be74218f6a820fb72f72ee7e05267083b4ac3b1ed456e0a290 F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70ee F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 @@ -1355,7 +1355,9 @@ F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3 F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de98745c F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e -F test/literal.test db71a7616dc027badf8d8f7ac6e23e8655915eb51b187c40a0a6e667aed495b7 +F test/literal.test e3d65d4091126cb008f31f57a324364511a83dd9461df31f60b5df6bd1f1f846 +F test/literal2.tcl 10713fed76885d43ada4161c0edf18fdc06e4405094e7baad888c6a1a29141b9 +F test/literal2.test 2e3ad4bc8cb019169d4f43400658fdc35acdcde7f79322c952b61ca07ac27ba6 F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db @@ -2159,8 +2161,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b57327be4be9638f5ee834a6c9e06135aaef9e499516f8b53f927ffe81033f61 -R dc78fe3596a95d4e421310d3dc095c60 -U drh -Z 8ae254187429ecb3341b495c0d5f92df +P 5bbc378a3022e2013048c8e296d95de21d15a8d7c1b62e8ffb64782f952cfe12 +R 5ae47ec02f1c44e5b63d2c7bab26a43e +U dan +Z 067422c390415b433adad2a7d339896e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 40427080bf..0b49600412 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5bbc378a3022e2013048c8e296d95de21d15a8d7c1b62e8ffb64782f952cfe12 \ No newline at end of file +b15bbb201a7727806b9fd6c02f2579b01e0e38155a8f9e24c5d6f487fe3e8b6c \ No newline at end of file diff --git a/src/tokenize.c b/src/tokenize.c index 37254da70d..7a30512ec6 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -477,8 +477,14 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; for(i+=2; 1; i++){ if( sqlite3Isdigit(z[i])==0 ){ - if( z[i]==SQLITE_DIGIT_SEPARATOR ){ *tokenType = TK_QNUMBER; } - else{ break; } + if( z[i]==SQLITE_DIGIT_SEPARATOR + && sqlite3Isdigit(z[i-1]) + && sqlite3Isdigit(z[i+1]) + ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } } } } diff --git a/test/literal.test b/test/literal.test index 3140eedf2a..fe6b70acf7 100644 --- a/test/literal.test +++ b/test/literal.test @@ -37,6 +37,10 @@ proc test_literal {tn lit type val} { " [list $type $val] } +proc test_literal_error {tn lit unrec} { + do_catchsql_test $tn "SELECT $lit" "1 {unrecognized token: \"$unrec\"}" +} + test_literal 1.0 45 integer 45 test_literal 1.1 0xFF integer 255 @@ -62,9 +66,31 @@ test_literal 3.1 1_000 integer 1000 test_literal 3.2 1.1_1 real 1.11 test_literal 3.3 1_0.1_1 real 10.11 test_literal 3.4 1e1_000 real Inf -test_literal 3.5 123______456.7_8__9_ real 123456.789 +test_literal 3.5 12_3_456.7_8_9 real 123456.789 test_literal 3.6 9_223_372_036_854_775_807 integer 9223372036854775807 test_literal 3.7 9_223_372_036_854_775_808 real 9.22337203685478e+18 test_literal 3.8 -9_223_372_036_854_775_808 integer -9223372036854775808 +foreach {tn lit unrec} { + 0 123a456 123a456 + 1 1_ 1_ + 2 1_.4 1_ + 3 1e_4 1e_4 + 4 1_e4 1_e4 + 5 1.4_e4 1.4_e4 + 6 1.4e+_4 1.4e + 7 1.4e-_4 1.4e + 8 1.4e4_ 1.4e4_ + 9 1.4_e4 1.4_e4 + 10 1.4e_4 1.4e_4 + 11 12__34 12__34 + 12 1234_ 1234_ + 13 12._34 12._34 + 14 12_.34 12_ + 15 12.34_ 12.34_ + 16 1.0e1_______2 1.0e1_______2 +} { + test_literal_error 4.$tn $lit $unrec +} + finish_test diff --git a/test/literal2.tcl b/test/literal2.tcl new file mode 100644 index 0000000000..2c97150b0c --- /dev/null +++ b/test/literal2.tcl @@ -0,0 +1,27 @@ +# 2018 May 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname $argv0] pg_common.tcl] + +#========================================================================= + + +start_test literal2 "2024 Jan 23" + +execsql_test 1.0 { SELECT 123_456 } +errorsql_test 1.1 { SELECT 123__456 } + +execsql_float_test 2.1 { SELECT 1.0e1_2 } + +finish_test + + diff --git a/test/literal2.test b/test/literal2.test new file mode 100644 index 0000000000..743df44eb7 --- /dev/null +++ b/test/literal2.test @@ -0,0 +1,48 @@ +# 2024 Jan 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +#################################################### +# DO NOT EDIT! THIS FILE IS AUTOMATICALLY GENERATED! +#################################################### + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix literal2 + +do_execsql_test 1.0 { + SELECT 123_456 +} {123456} + +# PG says ERROR: trailing junk after numeric literal at or near "123_" +do_test 1.1 { catch { execsql { + SELECT 123__456 +} } } 1 + + +do_test 2.1 { + set myres {} + foreach r [db eval {SELECT 1.0e1_2}] { + lappend myres [format %.4f [set r]] + } + set res2 {1000000000000.0000} + set i 0 + foreach r [set myres] r2 [set res2] { + if {[set r]<([set r2]-0.0001) || [set r]>([set r2]+0.0001)} { + error "list element [set i] does not match: got=[set r] expected=[set r2]" + } + incr i + } + set {} {} +} {} + +finish_test From 8374f7dfaf1106f01205e874b8b7e86a14f9d6cf Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 22 Jan 2024 19:38:55 +0000 Subject: [PATCH 6/8] Allow underscores to occur in hex literals. FossilOrigin-Name: 81a56229460cc5b6acfd3c3729fcf89ea3cccb546ca2b4f4035b140c60911e18 --- manifest | 20 +++++----- manifest.uuid | 2 +- src/tokenize.c | 96 ++++++++++++++++++++++++++-------------------- src/util.c | 10 +++-- test/literal2.tcl | 8 ++++ test/literal2.test | 18 +++++++++ test/misc1.test | 2 +- 7 files changed, 98 insertions(+), 58 deletions(-) diff --git a/manifest b/manifest index c1b974d23a..77f3ca0ff6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sfor\sthe\scode\son\sthis\sbranch. -D 2024-01-22T19:00:50.821 +C Allow\sunderscores\sto\soccur\sin\shex\sliterals. +D 2024-01-22T19:38:55.172 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -800,13 +800,13 @@ F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb7 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c f283d8017b57d8be74218f6a820fb72f72ee7e05267083b4ac3b1ed456e0a290 +F src/tokenize.c 3ea60fcd98a0eb1391592a080fb8871b0026eaffbdb5795ef3b19bb7d48e702a F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70ee F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e -F src/util.c 2b274b03f0d4fd1bf253f160d6b399bc5265338b043c078045c5e32298891ec8 +F src/util.c 25e0b776d5bb326d6009fbd8705c5ab332318b0f1c213dab4a717524defb9a7d F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 92910d536e0b77505599cd6ae5d9d449e4a5d31ada61da4c0bb84f6ccb2c3189 F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb @@ -1356,8 +1356,8 @@ F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de9874 F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e F test/literal.test e3d65d4091126cb008f31f57a324364511a83dd9461df31f60b5df6bd1f1f846 -F test/literal2.tcl 10713fed76885d43ada4161c0edf18fdc06e4405094e7baad888c6a1a29141b9 -F test/literal2.test 2e3ad4bc8cb019169d4f43400658fdc35acdcde7f79322c952b61ca07ac27ba6 +F test/literal2.tcl 70cf73b43f09396a4ba3ba7462b107a32866bb77deaa991c3296aa978edcdfdd +F test/literal2.test dd6b763feb086fa34b4229c527b35099020e7f5d588089fff9e4434c71695f60 F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db @@ -1410,7 +1410,7 @@ F test/minmax.test fe638b55d77d2375531a8f549b338eafcd9adfbd2f72df37ed77d9b26ca0a F test/minmax2.test cf9311babb6f0518d04e42fd6a42c619531c4309a9dd790a2c4e9b3bc595e0de F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 272ca395257f05937dc96441c9dde4bc9fbf116a8d4fa02baeb0d13d50e36c87 -F test/misc1.test 8d138a4926ab90617c1aa29ce26e7785ae2b83a4d3a195d543b7374e05589dd1 +F test/misc1.test e3e36262aff1bd9b8b9bf1eeb3af04adb3fc1e23f0a92dbff708bba9e939ace1 F test/misc2.test 71e746af479119386ac2ed7ab7d81d99970e75b49ffd3e8efffee100b4b5f350 F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d F test/misc4.test 10cd6addb2fa9093df4751a1b92b50440175dd5468a6ec84d0386e78f087db0e @@ -2161,8 +2161,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5bbc378a3022e2013048c8e296d95de21d15a8d7c1b62e8ffb64782f952cfe12 -R 5ae47ec02f1c44e5b63d2c7bab26a43e +P b15bbb201a7727806b9fd6c02f2579b01e0e38155a8f9e24c5d6f487fe3e8b6c +R d0d892d569c1b9650e2494b0fb9379db U dan -Z 067422c390415b433adad2a7d339896e +Z 87a049db1576d41d80b78f3d72e95d0a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0b49600412..bec89eba1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b15bbb201a7727806b9fd6c02f2579b01e0e38155a8f9e24c5d6f487fe3e8b6c \ No newline at end of file +81a56229460cc5b6acfd3c3729fcf89ea3cccb546ca2b4f4035b140c60911e18 \ No newline at end of file diff --git a/src/tokenize.c b/src/tokenize.c index 7a30512ec6..f01548d4cb 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -437,58 +437,70 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_INTEGER; #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ - for(i=3; sqlite3Isxdigit(z[i]); i++){} - return i; - } -#endif - for(i=0; 1; i++){ - if( sqlite3Isdigit(z[i])==0 ){ - if( z[i]==SQLITE_DIGIT_SEPARATOR - && sqlite3Isdigit(z[i-1]) - && sqlite3Isdigit(z[i+1]) - ){ - *tokenType = TK_QNUMBER; - }else{ - break; + for(i=3; 1; i++){ + if( sqlite3Isxdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR + && sqlite3Isxdigit(z[i-1]) + && sqlite3Isxdigit(z[i+1]) + ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + }else +#endif + { + for(i=0; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR + && sqlite3Isdigit(z[i-1]) + && sqlite3Isdigit(z[i+1]) + ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } } } - } #ifndef SQLITE_OMIT_FLOATING_POINT - if( z[i]=='.' ){ - if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; - for(i++; 1; i++){ - if( sqlite3Isdigit(z[i])==0 ){ - if( z[i]==SQLITE_DIGIT_SEPARATOR - && sqlite3Isdigit(z[i-1]) - && sqlite3Isdigit(z[i+1]) - ){ - *tokenType = TK_QNUMBER; - }else{ - break; + if( z[i]=='.' ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i++; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR + && sqlite3Isdigit(z[i-1]) + && sqlite3Isdigit(z[i+1]) + ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } } } } - } - if( (z[i]=='e' || z[i]=='E') && - ( sqlite3Isdigit(z[i+1]) - || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) - ) - ){ - if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; - for(i+=2; 1; i++){ - if( sqlite3Isdigit(z[i])==0 ){ - if( z[i]==SQLITE_DIGIT_SEPARATOR - && sqlite3Isdigit(z[i-1]) - && sqlite3Isdigit(z[i+1]) - ){ - *tokenType = TK_QNUMBER; - }else{ - break; + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i+=2; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR + && sqlite3Isdigit(z[i-1]) + && sqlite3Isdigit(z[i+1]) + ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } } } } - } #endif + } while( IdChar(z[i]) ){ *tokenType = TK_ILLEGAL; i++; diff --git a/src/util.c b/src/util.c index 24eff14e9f..9a2ea9e901 100644 --- a/src/util.c +++ b/src/util.c @@ -312,10 +312,9 @@ void sqlite3DequoteExpr(Expr *p){ } /* -** Expression p is a QINTEGER or QFLOAT (quoted integer or float). Dequote -** the value in p->u.zToken and set the type to INTEGER or FLOAT. "Quoted" -** integers or floats are those that contain '_' characters that must -** be removed before further processing. +** Expression p is a QNUMBER (quoted number). Dequote the value in p->u.zToken +** and set the type to INTEGER or FLOAT. "Quoted" integers or floats are those +** that contain '_' characters that must be removed before further processing. */ void sqlite3DequoteNumber(Expr *p){ if( p ){ @@ -329,6 +328,9 @@ void sqlite3DequoteNumber(Expr *p){ if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT; } }while( *pIn++ ); + if( p->u.zToken[0]=='0' && p->u.zToken[1]=='x' ){ + p->op = TK_INTEGER; + } } } diff --git a/test/literal2.tcl b/test/literal2.tcl index 2c97150b0c..029d33f51a 100644 --- a/test/literal2.tcl +++ b/test/literal2.tcl @@ -22,6 +22,14 @@ errorsql_test 1.1 { SELECT 123__456 } execsql_float_test 2.1 { SELECT 1.0e1_2 } + +execsql_test 3.0 { SELECT 0xFF_FF } +execsql_test 3.1 { SELECT 0xFF_EF } + +errorsql_test 3.2 { SELECT 0xFF__EF } +# errorsql_test 3.3 { SELECT 0x_FFEF } +errorsql_test 3.4 { SELECT 0xFFEF_ } + finish_test diff --git a/test/literal2.test b/test/literal2.test index 743df44eb7..eda526fcef 100644 --- a/test/literal2.test +++ b/test/literal2.test @@ -45,4 +45,22 @@ do_test 2.1 { set {} {} } {} +do_execsql_test 3.0 { + SELECT 0xFF_FF +} {65535} + +do_execsql_test 3.1 { + SELECT 0xFF_EF +} {65519} + +# PG says ERROR: trailing junk after numeric literal at or near "0xFF_" +do_test 3.2 { catch { execsql { + SELECT 0xFF__EF +} } } 1 + +# PG says ERROR: trailing junk after numeric literal at or near "0xFFEF_" +do_test 3.4 { catch { execsql { + SELECT 0xFFEF_ +} } } 1 + finish_test diff --git a/test/misc1.test b/test/misc1.test index 83acc752af..8110d38678 100644 --- a/test/misc1.test +++ b/test/misc1.test @@ -654,7 +654,7 @@ do_catchsql_test misc1-21.1 { } {1 {near "#0": syntax error}} do_catchsql_test misc1-21.2 { VALUES(0,0x0MATCH#0; -} {1 {near ";": syntax error}} +} {1 {unrecognized token: "0x0MATCH"}} # 2015-04-15 do_execsql_test misc1-22.1 { From a545de6827e9839da413f52e800f8184eedc6060 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 22 Jan 2024 19:42:56 +0000 Subject: [PATCH 7/8] Fix a problem in the previous commit with hex literals that start with "0X" instead of "0x". FossilOrigin-Name: c063c89b11487e6e712b97de604db316fa97bcf91ed810bb2dcbbcb54c68dbf4 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/util.c | 2 +- test/literal2.tcl | 15 ++++++++++----- test/literal2.test | 26 ++++++++++++++++++++++---- 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 77f3ca0ff6..f48cebcf73 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sunderscores\sto\soccur\sin\shex\sliterals. -D 2024-01-22T19:38:55.172 +C Fix\sa\sproblem\sin\sthe\sprevious\scommit\swith\shex\sliterals\sthat\sstart\swith\s"0X"\sinstead\sof\s"0x". +D 2024-01-22T19:42:56.220 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -806,7 +806,7 @@ F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e -F src/util.c 25e0b776d5bb326d6009fbd8705c5ab332318b0f1c213dab4a717524defb9a7d +F src/util.c ef37d377684d6f725773c15bfc1ef5b75483b4f3b6b6198d4b8b969831623be0 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 92910d536e0b77505599cd6ae5d9d449e4a5d31ada61da4c0bb84f6ccb2c3189 F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb @@ -1356,8 +1356,8 @@ F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de9874 F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e F test/literal.test e3d65d4091126cb008f31f57a324364511a83dd9461df31f60b5df6bd1f1f846 -F test/literal2.tcl 70cf73b43f09396a4ba3ba7462b107a32866bb77deaa991c3296aa978edcdfdd -F test/literal2.test dd6b763feb086fa34b4229c527b35099020e7f5d588089fff9e4434c71695f60 +F test/literal2.tcl 1499037beaf661aeecdbe48801220a181d805372a64c6128d5f26bb6a4a8f0ce +F test/literal2.test b149e16b5fc9ee6249069a8858ed41052f222014fe0ba7ad43c2fb989c2dada2 F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db @@ -2161,8 +2161,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b15bbb201a7727806b9fd6c02f2579b01e0e38155a8f9e24c5d6f487fe3e8b6c -R d0d892d569c1b9650e2494b0fb9379db +P 81a56229460cc5b6acfd3c3729fcf89ea3cccb546ca2b4f4035b140c60911e18 +R 8fee7748a406cec8844b4286c6b28b8f U dan -Z 87a049db1576d41d80b78f3d72e95d0a +Z 6bae334234c9987a328b3a9421e4a531 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bec89eba1b..de37a7858c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -81a56229460cc5b6acfd3c3729fcf89ea3cccb546ca2b4f4035b140c60911e18 \ No newline at end of file +c063c89b11487e6e712b97de604db316fa97bcf91ed810bb2dcbbcb54c68dbf4 \ No newline at end of file diff --git a/src/util.c b/src/util.c index 9a2ea9e901..ed7789591b 100644 --- a/src/util.c +++ b/src/util.c @@ -328,7 +328,7 @@ void sqlite3DequoteNumber(Expr *p){ if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT; } }while( *pIn++ ); - if( p->u.zToken[0]=='0' && p->u.zToken[1]=='x' ){ + if( p->u.zToken[0]=='0' && (p->u.zToken[1]=='x' || p->u.zToken[1]=='X') ){ p->op = TK_INTEGER; } } diff --git a/test/literal2.tcl b/test/literal2.tcl index 029d33f51a..e14a03587b 100644 --- a/test/literal2.tcl +++ b/test/literal2.tcl @@ -23,12 +23,17 @@ errorsql_test 1.1 { SELECT 123__456 } execsql_float_test 2.1 { SELECT 1.0e1_2 } -execsql_test 3.0 { SELECT 0xFF_FF } -execsql_test 3.1 { SELECT 0xFF_EF } +execsql_test 3.0.0 { SELECT 0xFF_FF } +execsql_test 3.0.1 { SELECT 0xFF_EF } +errorsql_test 3.0.2 { SELECT 0xFF__EF } +# errorsql_test 3.0.3 { SELECT 0x_FFEF } +errorsql_test 3.0.4 { SELECT 0xFFEF_ } -errorsql_test 3.2 { SELECT 0xFF__EF } -# errorsql_test 3.3 { SELECT 0x_FFEF } -errorsql_test 3.4 { SELECT 0xFFEF_ } +execsql_test 3.1.0 { SELECT 0XFF_FF } +execsql_test 3.1.1 { SELECT 0XFF_EF } +errorsql_test 3.1.2 { SELECT 0XFF__EF } +# errorsql_test 3.1.3 { SELECT 0X_FFEF } +errorsql_test 3.1.4 { SELECT 0XFFEF_ } finish_test diff --git a/test/literal2.test b/test/literal2.test index eda526fcef..ed177ca261 100644 --- a/test/literal2.test +++ b/test/literal2.test @@ -45,22 +45,40 @@ do_test 2.1 { set {} {} } {} -do_execsql_test 3.0 { +do_execsql_test 3.0.0 { SELECT 0xFF_FF } {65535} -do_execsql_test 3.1 { +do_execsql_test 3.0.1 { SELECT 0xFF_EF } {65519} # PG says ERROR: trailing junk after numeric literal at or near "0xFF_" -do_test 3.2 { catch { execsql { +do_test 3.0.2 { catch { execsql { SELECT 0xFF__EF } } } 1 # PG says ERROR: trailing junk after numeric literal at or near "0xFFEF_" -do_test 3.4 { catch { execsql { +do_test 3.0.4 { catch { execsql { SELECT 0xFFEF_ } } } 1 +do_execsql_test 3.1.0 { + SELECT 0XFF_FF +} {65535} + +do_execsql_test 3.1.1 { + SELECT 0XFF_EF +} {65519} + +# PG says ERROR: trailing junk after numeric literal at or near "0XFF_" +do_test 3.1.2 { catch { execsql { + SELECT 0XFF__EF +} } } 1 + +# PG says ERROR: trailing junk after numeric literal at or near "0XFFEF_" +do_test 3.1.4 { catch { execsql { + SELECT 0XFFEF_ +} } } 1 + finish_test From 406eb5a146e06c29a5f1d8b8e25c10d125945a20 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 23 Jan 2024 11:20:58 +0000 Subject: [PATCH 8/8] Add extra checks for the validity of a numeric literal to sqlite3DequoteNumber(). FossilOrigin-Name: d57407ef59baf699b72c8c4859abfaa7977dd41f6f16eb8fe1d53a68806eb966 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/parse.y | 2 +- src/sqliteInt.h | 2 +- src/tokenize.c | 20 ++++---------------- src/util.c | 13 +++++++++---- test/literal.test | 4 ++-- 7 files changed, 28 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index f48cebcf73..0470957ccf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\sprevious\scommit\swith\shex\sliterals\sthat\sstart\swith\s"0X"\sinstead\sof\s"0x". -D 2024-01-22T19:42:56.220 +C Add\sextra\schecks\sfor\sthe\svalidity\sof\sa\snumeric\sliteral\sto\ssqlite3DequoteNumber(). +D 2024-01-23T11:20:58.812 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -727,7 +727,7 @@ F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a -F src/parse.y 2354aaf964e7c4154a9dbe56ea55a797a0fa3021c38b50afe491ea4a387bf971 +F src/parse.y d2823ae4a503f83b3e8629c31470686624e46851d576c1f2b7bbec2e7328bb05 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 @@ -743,7 +743,7 @@ F src/shell.c.in d1ed426aae2d547932971e8019939cacb4dfda8258e45b8924b250e488e2d53 F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 6e5e330d84b4ace70e3163721601f01df84566e6db21e1fc45bd00636e3d6640 +F src/sqliteInt.h a1367a4cd90c90e5eb2e5ca7d1be96823507f63b43030deb394f90f4f8d9ac10 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -800,13 +800,13 @@ F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb7 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 3ea60fcd98a0eb1391592a080fb8871b0026eaffbdb5795ef3b19bb7d48e702a +F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68 F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70ee F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e -F src/util.c ef37d377684d6f725773c15bfc1ef5b75483b4f3b6b6198d4b8b969831623be0 +F src/util.c 88484a62e2465728288ca6e5b10f30489058e66266f52c70e87663310298793b F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 92910d536e0b77505599cd6ae5d9d449e4a5d31ada61da4c0bb84f6ccb2c3189 F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb @@ -1355,7 +1355,7 @@ F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3 F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de98745c F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e -F test/literal.test e3d65d4091126cb008f31f57a324364511a83dd9461df31f60b5df6bd1f1f846 +F test/literal.test c4f6f281964ac5ab48a32bd978e80644affac822664879d7558762b2fad7aff5 F test/literal2.tcl 1499037beaf661aeecdbe48801220a181d805372a64c6128d5f26bb6a4a8f0ce F test/literal2.test b149e16b5fc9ee6249069a8858ed41052f222014fe0ba7ad43c2fb989c2dada2 F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 @@ -2161,8 +2161,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 81a56229460cc5b6acfd3c3729fcf89ea3cccb546ca2b4f4035b140c60911e18 -R 8fee7748a406cec8844b4286c6b28b8f +P c063c89b11487e6e712b97de604db316fa97bcf91ed810bb2dcbbcb54c68dbf4 +R 92b3c6e9a0a36e247ccad3d6a5b99b6c U dan -Z 6bae334234c9987a328b3a9421e4a531 +Z 62284248efaccf35cb9923c551aa2ba1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index de37a7858c..66162263f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c063c89b11487e6e712b97de604db316fa97bcf91ed810bb2dcbbcb54c68dbf4 \ No newline at end of file +d57407ef59baf699b72c8c4859abfaa7977dd41f6f16eb8fe1d53a68806eb966 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index b0d03a04ee..c26a9bccc2 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1926,7 +1926,7 @@ filter_clause(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } term(A) ::= QNUMBER(X). { A=tokenExpr(pParse,@X,X); - sqlite3DequoteNumber(A); + sqlite3DequoteNumber(pParse, A); } /* There must be no more than 255 tokens defined above. If this grammar diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c4aaf9d203..2db491fc81 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4794,7 +4794,7 @@ int sqlite3ErrorToParser(sqlite3*,int); void sqlite3Dequote(char*); void sqlite3DequoteExpr(Expr*); void sqlite3DequoteToken(Token*); -void sqlite3DequoteNumber(Expr*); +void sqlite3DequoteNumber(Parse*, Expr*); void sqlite3TokenInit(Token*,char*); int sqlite3KeywordCode(const unsigned char*, int); int sqlite3RunParser(Parse*, const char*); diff --git a/src/tokenize.c b/src/tokenize.c index f01548d4cb..65d1fbf350 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -439,10 +439,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ for(i=3; 1; i++){ if( sqlite3Isxdigit(z[i])==0 ){ - if( z[i]==SQLITE_DIGIT_SEPARATOR - && sqlite3Isxdigit(z[i-1]) - && sqlite3Isxdigit(z[i+1]) - ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ *tokenType = TK_QNUMBER; }else{ break; @@ -454,10 +451,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ { for(i=0; 1; i++){ if( sqlite3Isdigit(z[i])==0 ){ - if( z[i]==SQLITE_DIGIT_SEPARATOR - && sqlite3Isdigit(z[i-1]) - && sqlite3Isdigit(z[i+1]) - ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ *tokenType = TK_QNUMBER; }else{ break; @@ -469,10 +463,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; for(i++; 1; i++){ if( sqlite3Isdigit(z[i])==0 ){ - if( z[i]==SQLITE_DIGIT_SEPARATOR - && sqlite3Isdigit(z[i-1]) - && sqlite3Isdigit(z[i+1]) - ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ *tokenType = TK_QNUMBER; }else{ break; @@ -488,10 +479,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; for(i+=2; 1; i++){ if( sqlite3Isdigit(z[i])==0 ){ - if( z[i]==SQLITE_DIGIT_SEPARATOR - && sqlite3Isdigit(z[i-1]) - && sqlite3Isdigit(z[i+1]) - ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ *tokenType = TK_QNUMBER; }else{ break; diff --git a/src/util.c b/src/util.c index ed7789591b..5a88979fe4 100644 --- a/src/util.c +++ b/src/util.c @@ -316,21 +316,26 @@ void sqlite3DequoteExpr(Expr *p){ ** and set the type to INTEGER or FLOAT. "Quoted" integers or floats are those ** that contain '_' characters that must be removed before further processing. */ -void sqlite3DequoteNumber(Expr *p){ +void sqlite3DequoteNumber(Parse *pParse, Expr *p){ if( p ){ const char *pIn = p->u.zToken; char *pOut = p->u.zToken; + int bHex = (pIn[0]=='0' && (pIn[1]=='x' || pIn[1]=='X')); assert( p->op==TK_QNUMBER ); p->op = TK_INTEGER; do { if( *pIn!=SQLITE_DIGIT_SEPARATOR ){ *pOut++ = *pIn; if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT; + }else{ + if( (bHex==0 && (!sqlite3Isdigit(pIn[-1]) || !sqlite3Isdigit(pIn[1]))) + || (bHex==1 && (!sqlite3Isxdigit(pIn[-1]) || !sqlite3Isxdigit(pIn[1]))) + ){ + sqlite3ErrorMsg(pParse, "unrecognized token: \"%s\"", p->u.zToken); + } } }while( *pIn++ ); - if( p->u.zToken[0]=='0' && (p->u.zToken[1]=='x' || p->u.zToken[1]=='X') ){ - p->op = TK_INTEGER; - } + if( bHex ) p->op = TK_INTEGER; } } diff --git a/test/literal.test b/test/literal.test index fe6b70acf7..30205692c9 100644 --- a/test/literal.test +++ b/test/literal.test @@ -74,7 +74,7 @@ test_literal 3.8 -9_223_372_036_854_775_808 integer -9223372036854775808 foreach {tn lit unrec} { 0 123a456 123a456 1 1_ 1_ - 2 1_.4 1_ + 2 1_.4 1_.4 3 1e_4 1e_4 4 1_e4 1_e4 5 1.4_e4 1.4_e4 @@ -86,7 +86,7 @@ foreach {tn lit unrec} { 11 12__34 12__34 12 1234_ 1234_ 13 12._34 12._34 - 14 12_.34 12_ + 14 12_.34 12_.34 15 12.34_ 12.34_ 16 1.0e1_______2 1.0e1_______2 } {