diff --git a/Makefile.in b/Makefile.in index c521586a14..6b17d43997 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1135,19 +1135,21 @@ keywordhash.h: $(TOP)/tool/mkkeywordhash.c # Source files that go into making shell.c SHELL_SRC = \ $(TOP)/src/shell.c.in \ - $(TOP)/ext/misc/appendvfs.c \ + $(TOP)/ext/consio/console_io.c \ + $(TOP)/ext/consio/console_io.h \ + $(TOP)/ext/misc/appendvfs.c \ $(TOP)/ext/misc/completion.c \ - $(TOP)/ext/misc/decimal.c \ - $(TOP)/ext/misc/basexx.c \ - $(TOP)/ext/misc/base64.c \ - $(TOP)/ext/misc/base85.c \ + $(TOP)/ext/misc/decimal.c \ + $(TOP)/ext/misc/basexx.c \ + $(TOP)/ext/misc/base64.c \ + $(TOP)/ext/misc/base85.c \ $(TOP)/ext/misc/fileio.c \ - $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/regexp.c \ - $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/shathree.c \ $(TOP)/ext/misc/sqlar.c \ - $(TOP)/ext/misc/uint.c \ + $(TOP)/ext/misc/uint.c \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ @@ -1156,7 +1158,7 @@ SHELL_SRC = \ $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/sqlite3recover.c \ $(TOP)/ext/recover/sqlite3recover.h \ - $(TOP)/src/test_windirent.c + $(TOP)/src/test_windirent.c shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl has_tclsh84 $(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c diff --git a/Makefile.msc b/Makefile.msc index 4339be891f..3a4d46b01f 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2261,6 +2261,8 @@ keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe # Source files that go into making shell.c SHELL_SRC = \ $(TOP)\src\shell.c.in \ + $(TOP)\ext\consio\console_io.c \ + $(TOP)\ext\consio\console_io.h \ $(TOP)\ext\misc\appendvfs.c \ $(TOP)\ext\misc\completion.c \ $(TOP)\ext\misc\base64.c \ diff --git a/ext/consio/console_io.c b/ext/consio/console_io.c index 06db83aacc..c60e2b0a49 100755 --- a/ext/consio/console_io.c +++ b/ext/consio/console_io.c @@ -18,19 +18,25 @@ #endif #ifndef SHELL_NO_SYSINC -#include -#include -#include "console_io.h" -#include "sqlite3.h" +# include +# include +# include +# include +# include "console_io.h" +# include "sqlite3.h" #endif #if defined(_WIN32) || defined(WIN32) # ifndef SHELL_NO_SYSINC # include # include +# undef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# include # endif # ifdef SHELL_LEGACY_CONSOLE_IO # define SHELL_CON_TRANSLATE 2 /* Use UTF-8/MBCS translation for console I/O */ +extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int); # else # define SHELL_CON_TRANSLATE 1 /* Use WCHAR Windows APIs for console I/O */ # endif @@ -144,7 +150,7 @@ consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ){ } if( ix > 0 ) fflush(apf[ix]); #if SHELL_CON_TRANSLATE == 2 - _setmode(_fileno(apf[ix]), _O_U8TEXT); + // _setmode(_fileno(apf[ix]), _O_U8TEXT); _setmode(_fileno(apf[ix]), _O_TEXT); #endif } @@ -159,14 +165,14 @@ SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ){ if( consoleInfo.cscs & (CSCS_InConsole<hx, ppst->consMode); @@ -181,7 +187,6 @@ SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ){ } } } -#undef SHELL_INVALID_FILE_PTR static short isConOut(FILE *pf){ if( pf==consoleInfo.pst[1].pf ) return 1; @@ -207,45 +212,75 @@ SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *pf, short bFlush){ } #undef setModeFlushQ -SQLITE_INTERNAL_LINKAGE int fprintfUtf8(FILE *pfO, const char *zFormat, ...){ - va_list ap; +#if SHELL_CON_TRANSLATE +/* For fprintfUtf8() and printfUtf8() when stream is known as console. */ +static int conioVmPrintf(int rch, const char *zFormat, va_list ap){ int rv = 0; -#if SHELL_CON_TRANSLATE - short on = isConOut(pfO); -#endif - va_start(ap, zFormat); -#if SHELL_CON_TRANSLATE - if( on > 0 ){ - char *z1 = sqlite3_vmprintf(zFormat, ap); + char *z1 = sqlite3_vmprintf(zFormat, ap); # if SHELL_CON_TRANSLATE == 2 + if( z1!=NULL ){ + UINT ccp = GetConsoleOutputCP(); + FILE *pfO = consoleInfo.pst[rch].pf; /* Legacy translation to active code page, then MBCS chars out. */ char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); if( z2!=NULL ){ rv = (int)strlen(z2); - vfprintf(pfO, "%s", z2); + fputs(z2, pfO); sqlite3_free(z2); } -# else - /* Translation from UTF-8 to UTF-16, then WCHAR characters out. */ - if( z1!=NULL ){ - int nwc; - WCHAR *zw2 = 0; - rv = (int)strlen(z1); - nwc = MultiByteToWideChar(CP_UTF8,0,z1,rv,0,0); - if( nwc>0 ){ - zw2 = sqlite3_malloc64((nwc+1)*sizeof(WCHAR)); - if( zw2!=NULL ){ - HANDLE ho = (on==1)? consoleInfo.pst[1].hx : consoleInfo.pst[2].hx; - nwc = MultiByteToWideChar(CP_UTF8,0,z1,rv,zw2,nwc); - zw2[nwc] = 0; - WriteConsoleW(ho, zw2, nwc, 0, NULL); - sqlite3_free(zw2); - }else rv = 0; - } - } -# endif sqlite3_free(z1); + } +# elif SHELL_CON_TRANSLATE == 1 + /* Translation from UTF-8 to UTF-16, then WCHAR characters out. */ + if( z1!=NULL ){ + int nwc; + WCHAR *zw2 = 0; + rv = (int)strlen(z1); + nwc = MultiByteToWideChar(CP_UTF8,0,z1,rv,0,0); + if( nwc>0 ){ + zw2 = sqlite3_malloc64((nwc+1)*sizeof(WCHAR)); + if( zw2!=NULL ){ + HANDLE ho = consoleInfo.pst[rch].hx; + nwc = MultiByteToWideChar(CP_UTF8,0,z1,rv,zw2,nwc); + zw2[nwc] = 0; + WriteConsoleW(ho, zw2, nwc, 0, NULL); + sqlite3_free(zw2); + }else rv = 0; + } + sqlite3_free(z1); + } +# endif + return rv; +} +#endif + +SQLITE_INTERNAL_LINKAGE int printfUtf8(const char *zFormat, ...){ + va_list ap; + int rv; + va_start(ap, zFormat); +#if SHELL_CON_TRANSLATE + if( SHELL_INVALID_FILE_PTR != consoleInfo.pst[1].pf ){ + rv = conioVmPrintf(1, zFormat, ap); }else{ +#endif + rv = vfprintf(stdout, zFormat, ap); +#if SHELL_CON_TRANSLATE + } +#endif + va_end(ap); + return rv; +} +#undef SHELL_INVALID_FILE_PTR + +SQLITE_INTERNAL_LINKAGE int fprintfUtf8(FILE *pfO, const char *zFormat, ...){ + va_list ap; + int rv; + va_start(ap, zFormat); +#if SHELL_CON_TRANSLATE + short rch = isConOut(pfO); + if( rch > 0 ){ + rv = conioVmPrintf(rch, zFormat, ap); + }else { #endif rv = vfprintf(pfO, zFormat, ap); #if SHELL_CON_TRANSLATE @@ -263,11 +298,25 @@ SQLITE_INTERNAL_LINKAGE int fputsUtf8(const char *z, FILE *pfO){ #endif } +#if SHELL_CON_TRANSLATE==2 +static int mbcsToUtf8InPlaceIfValid(char *pc, int nci, int nco, UINT codePage){ + WCHAR wcOneCode[2]; + int nuo = 0; + int nwConvert = MultiByteToWideChar(codePage, MB_ERR_INVALID_CHARS, + pc, nci, wcOneCode, 2); + if( nwConvert > 0 ){ + nuo = WideCharToMultiByte(CP_UTF8, 0, wcOneCode, nwConvert, pc, nco, 0,0); + } + return nuo; +} +#endif + SQLITE_INTERNAL_LINKAGE char* fgetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ if( pfIn==0 ) pfIn = stdin; #if SHELL_CON_TRANSLATE if( pfIn == consoleInfo.pst[0].pf ){ -#define SHELL_GULP 150 /* Count of WCHARS to be gulped at a time */ +# if SHELL_CON_TRANSLATE==1 +# define SHELL_GULP 150 /* Count of WCHARS to be gulped at a time */ WCHAR wcBuf[SHELL_GULP+1]; int lend = 0, noc = 0; if( consoleInfo.stdinEof ) return 0; @@ -275,7 +324,7 @@ SQLITE_INTERNAL_LINKAGE char* fgetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ while( noc < ncMax-8-1 && !lend ){ /* There is room for at least 2 more characters and a 0-terminator. */ int na = (ncMax > SHELL_GULP*4+1 + noc)? SHELL_GULP : (ncMax-1 - noc)/4; -#undef SHELL_GULP +# undef SHELL_GULP DWORD nbr = 0; BOOL bRC = ReadConsoleW(consoleInfo.pst[0].hx, wcBuf, na, &nbr, 0); if( bRC && nbr>0 && (wcBuf[nbr-1]&0xF800)==0xD800 ){ @@ -304,7 +353,7 @@ SQLITE_INTERNAL_LINKAGE char* fgetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ } } } - /* Check for ^Z (anywhere in line) too. */ + /* Check for ^Z (anywhere in line) too, to act as EOF. */ while( iseg < noc ){ if( cBuf[iseg]==0x1a ){ consoleInfo.stdinEof = 1; @@ -320,6 +369,52 @@ SQLITE_INTERNAL_LINKAGE char* fgetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ if( noc == 0 ) return 0; cBuf[noc] = 0; return cBuf; +# elif SHELL_CON_TRANSLATE==2 + /* This is not done efficiently because it may never be used. + ** Also, it is interactive input so it need not be fast. */ + int nco = 0; + /* For converstion to WCHAR, or pre-test of same. */ + UINT ccp = GetConsoleCP(); /* For translation from mbcs. */ + if( ccp == CP_UTF8 ) return fgets(cBuf, ncMax, pfIn); + while( ncMax-nco >= 5 ){ + /* Have space for max UTF-8 group and 0-term. */ + int nug = 0; + int c = getc(pfIn); + if( c < 0 ){ + if( nco > 0 ) break; + else return 0; + } + cBuf[nco] = (char)c; + if( c < 0x80 ){ + ++nco; + if( c == '\n' ) break; + continue; + } + /* Deal with possible mbcs lead byte. */ + nug = mbcsToUtf8InPlaceIfValid(cBuf+nco, 1, ncMax-nco-1, ccp); + if( nug > 0 ){ + nco += nug; + }else{ + /* Must have just mbcs lead byte; get the trail byte. */ + int ct = getc(pfIn); + if( ct < 0 || ct == '\n' ){ + /* Just drop whatever garbage preceded the newline or. + ** EOF. It's not valid, should not happen, and there + ** is no good way to deal with it, short of bailing. */ + if( ct > 0 ){ + cBuf[nco++] = (int)ct; + } + break; + } + /* Treat ct as bona fide MBCS trailing byte, if valid. */ + cBuf[nco+1] = ct; + nug = mbcsToUtf8InPlaceIfValid(cBuf+nco, 2, ncMax-nco-1, ccp); + nco += nug; + } + } + cBuf[nco] = 0; + return cBuf; +# endif }else{ #endif return fgets(cBuf, ncMax, pfIn); diff --git a/ext/consio/console_io.h b/ext/consio/console_io.h index 2cbc2c1f97..e799c71e74 100644 --- a/ext/consio/console_io.h +++ b/ext/consio/console_io.h @@ -8,7 +8,7 @@ ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -************************************************************************* +******************************************************************************** ** This file exposes various interfaces used for console I/O by the ** SQLite project command-line tools. These interfaces are used at ** either source conglomeration time, compilation time, or run time. @@ -27,15 +27,6 @@ #ifndef SQLITE_INTERNAL_LINKAGE # define SQLITE_INTERNAL_LINKAGE extern /* external to translation unit */ # include -# include -# include -# if defined(_WIN32) || defined(WIN32) -# undef WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN -# include -# include -# include -# endif #else # define SHELL_NO_SYSINC /* Better yet, modify mkshellc.tcl for this. */ #endif @@ -98,6 +89,8 @@ SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ); ** translation as specified by set{Binary,Text}Mode(). */ SQLITE_INTERNAL_LINKAGE int fprintfUtf8(FILE *pfO, const char *zFormat, ...); +/* Like fprintfUtf8 except stream is always the recorded output. */ +SQLITE_INTERNAL_LINKAGE int printfUtf8(const char *zFormat, ...); /* ** Render output like fputs(). If the output is going to the diff --git a/manifest b/manifest index 6fec7ee4b3..a2db5a7373 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Move\sconsole_io\slib\sto\sits\sown\ssubdirectory,\setc/consio\s. -D 2023-11-06T15:31:08.971 +C Get\sdependencies\sinto\smake\srecipes.\sGet\slegacy\sconsole\sI/O\s(-DSHELL_LEGACY_CONSOLE_IO)\sworking.\sDue\sto\smovement\sof\sMBCS/UTF-8\stranslation\sinto\straditional\sstream\sI/O\ssimulacra,\sthe\sinput\stranslation\sdoes\snot\shappen\sthe\ssame\sway.\s(It\sworks\sthe\ssame,\sbut\sfails\sdifferently\sand\sa\sbit\sbetter.)\sAdded\sprintf()\sand\sfputs()\slook-alikes,\sand\smade\sCLI\suse\sthem. +D 2023-11-07T02:41:46.723 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 8b59912fc1538f96a08555605c5886cdcc733696ae7f22e374b2a4752196ca20 +F Makefile.in f1ec22cfe53927eb9411ed82e452a0984df0ef88acb7a45024d98c5b6909da12 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc f0cf219350d9af4fba411b4f6306dce2adc897484e8f446de1fb4f40de674d00 +F Makefile.msc 59bb36dba001f0b38212be0794fb838f25371008b180929bcf08aa799694c168 F README.md 963d30019abf0cc06b263cd2824bce022893f3f93a531758f6f04ff2194a16a8 F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -50,8 +50,8 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94 F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a -F ext/consio/console_io.c 7c9d44ae378855778b7c4bb171df336e5f0e629107e56d93fcaaa41b512d253d x src/console_io.c -F ext/consio/console_io.h 92a765da11922b57c899132fd31fb9f6a54802d9ca2489e618a66b7d5334acb9 w src/console_io.h +F ext/consio/console_io.c 5011c039c6224831ebfa7f6522cf4bce72229f50c45c9aa66df0a45acd4690bf x +F ext/consio/console_io.h e6055b6a13a2a9f237e1672f9ef861126a37a61db0e6218a137832557f10ea25 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 F ext/expert/expert1.test 0dd5cb096d66bed593e33053a3b364f6ef52ed72064bf5cf298364636dbf3cd6 @@ -727,7 +727,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c a19daa26e95f7245106a31f288b2f50c72d1f2cc156703f04c8c91450e111515 -F src/shell.c.in 012b84f928dc883b589e27086540df7317b8b465d366b6baac2717b07f59ee0c +F src/shell.c.in b651e2c297bfef8bd063159765b4ffab14f27816cb373b4995a4b411c33ecd51 F src/sqlite.h.in ef0e41e83ad1ac0dcc9ec9939bf541a44b1c5de821bee2d6c61754c3252f3276 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2144,8 +2144,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 58815f0ad259599a674e7d0fe0d4676a1c4fb8e118c08b619580ff655fb3be40 -R 9369b16a7548c239ad37858f67dafede +P 1d0583f2eb69fdca1cbc55763c0e86a7e32cb7771bfbc2cdf02da4e3fedbfa23 +R 8fe955210f7825f8cd02e07383e28c60 U larrybr -Z ed8eeffcaf90e73880f84d59f5812858 +Z 407fd2a55e54d86f679485ba9f27b473 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 55b7016ccf..d95739e422 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d0583f2eb69fdca1cbc55763c0e86a7e32cb7771bfbc2cdf02da4e3fedbfa23 \ No newline at end of file +1721dc6a434361c4e2b87c6e677b6dc223432b3cdd5b9eecabaa258889fb2d2a \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index b4a0d1936f..426401e671 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -567,8 +567,9 @@ static char *dynamicContinuePrompt(void){ /* From here onward, fgets() is redirected to the console_io library. */ #define fgets(b,n,f) fgetsUtf8(b,n,f) -/* And, (varargs) utf8_printf(f,z,...) is redirected to the same. */ +/* Also, (varargs) utf8_printf(f,z,...),printf(z,...) so redirected. */ #define utf8_printf fprintfUtf8 +#define printf printfUtf8 /* And, utf8_print(f,z) is redirected to fputsUtf8(z,f) in the library. */ #define utf8_print(f,z) fputsUtf8(z,f) @@ -12427,8 +12428,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char *zHistory; const char *zCharset = ""; int nHistory; -#if SHELL_CON_TRANSLATE +#if SHELL_CON_TRANSLATE==1 zCharset = " (UTF-16 console I/O)"; +#elif SHELL_CON_TRANSLATE==2 + zCharset = " (MBCS console I/O)"; #endif printf( "SQLite version %s %.19s%s\n" /*extra-version-info*/