The tokenizer should never return a negative size of the next token.

Ticket #453. (CVS 1098)

FossilOrigin-Name: 4fbca3ab09596c530da7c50657f3bc9140178dd5

manifest

@@ -1,5 +1,5 @@
-C Add\stest\scases\sthat\scheck\sout\ssqlite_get_table()\son\squeries\swith\sresults\ssets\ncontaining\s100\scolumns.\s(CVS\s1097)
-D 2003-09-09T00:47:47
+C The\stokenizer\sshould\snever\sreturn\sa\snegative\ssize\sof\sthe\snext\stoken.\nTicket\s#453.\s(CVS\s1098)
+D 2003-09-12T02:08:15
 F Makefile.in 0cf2ffb6dc35694895e0dac488bc1259b6a4eb90
 F Makefile.linux-gcc b86a99c493a5bfb402d1d9178dcdc4bd4b32f906
 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
@@ -56,7 +56,7 @@ F src/test1.c f9d5816610f7ec4168ab7b098d5207a5708712b6
 F src/test2.c 5014337d8576b731cce5b5a14bec4f0daf432700
 F src/test3.c 30985ebdfaf3ee1462a9b0652d3efbdc8d9798f5
 F src/threadtest.c d641a5219e718e18a1a80a50eb9bb549f451f42e
-F src/tokenize.c 74152bde07da0623aaa60fb6ab71d5af1c035546
+F src/tokenize.c 8df640021c7a18f75411a8fd25adbb76bf528e7e
 F src/trigger.c 474581eaab388233df01bb019e558af2965decbf
 F src/update.c 24260b4fda00c9726d27699a0561d53c0dccc397
 F src/util.c f16efa2d60bfd4e31ae06b07ed149557e828d294
@@ -102,7 +102,7 @@ F test/memdb.test 6ece25c7c0e6500199d3662607a3edca081abb2a
 F test/memleak.test a18e6810cae96d2f6f5136920267adbefc8e1e90
 F test/minmax.test 6d9b6d6ee34f42e2a58dffece1f76d35f446b3af
 F test/misc1.test 0b98d493b0cf55cb5f53e1f3df8107c166eecb5a
-F test/misc2.test d7bc75fae9157c4a3f4914b505713f398b3c5422
+F test/misc2.test 5818bfafd07535b0437f15c32bed983f3bd363b2
 F test/misuse.test a3aa2b18a97e4c409a1fcaff5151a4dd804a0162
 F test/notnull.test 7a08117a71e74b0321aaa937dbeb41a09d6eb1d0
 F test/null.test 5c2b57307e4b6178aae825eb65ddbee01e76b0fd
@@ -173,7 +173,7 @@ F www/speed.tcl 2f6b1155b99d39adb185f900456d1d592c4832b3
 F www/sqlite.tcl 3c83b08cf9f18aa2d69453ff441a36c40e431604
 F www/tclsqlite.tcl b9271d44dcf147a93c98f8ecf28c927307abd6da
 F www/vdbe.tcl 9b9095d4495f37697fd1935d10e14c6015e80aa1
-P f6c4908e9b5b6ac9adc4af50dc5110dbb655dae3
-R f7938d35621546ffe8cfa85d2210afbb
+P f9feedec32c975235c0d0996962033f6a63312ba
+R f36ba4593d56d9555b41bc6f7efad70c
 U drh
-Z d4864a97177572787e9b3823564d9aff
+Z 3e7100d230b7f9eaf7343d1e1cea1236