Provide SQLITE_U8TEXT_ONLY and SQLITE_U8TEXT_STDIO compile-time options

to the sqlite3_stdio.c module.

FossilOrigin-Name: f31588520e3f45b50dcaa9eecab17f52ebb56bb53d0f9bdb88cc596d1a156353

ext/misc/sqlite3_stdio.c

@@ -30,6 +30,43 @@
 #include <io.h>
 #include <fcntl.h>
 
+/*
+** If the SQLITE_U8TEXT_ONLY option is defined, then only use
+** _O_U8TEXT, _O_WTEXT, and similar together with the UTF-16
+** interfaces to the Windows CRT.  The use of ANSI-only routines
+** like fputs() and ANSI modes like _O_TEXT and _O_BINARY is
+** avoided.
+**
+** The downside of using SQLITE_U8TEXT_ONLY is that it becomes
+** impossible to output a bare newline character (0x0a) - that is,
+** a newline that is not preceded by a carriage return (0x0d).
+** And without that capability, sometimes the output will be slightly
+** incorrect, as extra 0x0d characters will have been inserted where
+** they do not belong.
+**
+** The SQLITE_U8TEXT_STDIO compile-time option is a compromise.
+** It always enables _O_WTEXT or similar for stdin, stdout, stderr,
+** but allows other streams to be _O_TEXT and/or O_BINARY.  The
+** SQLITE_U8TEXT_STDIO option has the same downside as SQLITE_U8TEXT_ONLY
+** in that stray 0x0d characters might appear where they ought not, but
+** at least with this option those characters only appear on standard
+** I/O streams, and not on new streams that might be created by the
+** application using sqlite3_fopen() or sqlite3_popen().
+*/
+#if defined(SQLITE_U8TEXT_ONLY)
+# define UseWtextForOutput(fd) 1
+# define UseWtextForInput(fd)  1
+# define IsConsole(fd)         _isatty(_fileno(fd))
+#elif defined(SQLITE_U8TEXT_STDIO)
+# define UseWtextForOutput(fd) ((fd)==stdout || (fd)==stderr)
+# define UseWtextForInput(fd)  ((fd)==stdin)
+# define IsConsole(fd)         _isatty(_fileno(fd))
+#else
+# define UseWtextForOutput(fd) _isatty(_fileno(fd))
+# define UseWtextForInput(fd)  _isatty(_fileno(fd))
+# define IsConsole(fd)         1
+#endif
+
 /*
 ** Work-alike for the fopen() routine from the standard C library.
 */
@@ -83,7 +120,7 @@ FILE *sqlite3_popen(const char *zCommand, const char *zMode){
 ** Work-alike for fgets() from the standard C library.
 */
 char *sqlite3_fgets(char *buf, int sz, FILE *in){
-  if( _isatty(_fileno(in)) ){
+  if( UseWtextForInput(in) ){
     /* When reading from the command-prompt in Windows, it is necessary
     ** to use _O_WTEXT input mode to read UTF-16 characters, then translate
     ** that into UTF-8.  Otherwise, non-ASCII characters all get translated
@@ -91,7 +128,7 @@ char *sqlite3_fgets(char *buf, int sz, FILE *in){
     */
     wchar_t *b1 = malloc( sz*sizeof(wchar_t) );
     if( b1==0 ) return 0;
-    _setmode(_fileno(in), _O_WTEXT);
+    _setmode(_fileno(in), IsConsole(in) ? _O_WTEXT : _O_U8TEXT);
     if( fgetws(b1, sz/4, in)==0 ){
       sqlite3_free(b1);
       return 0;
@@ -110,7 +147,7 @@ char *sqlite3_fgets(char *buf, int sz, FILE *in){
 ** Work-alike for fputs() from the standard C library.
 */
 int sqlite3_fputs(const char *z, FILE *out){
-  if( _isatty(_fileno(out)) ){
+  if( UseWtextForOutput(out) ){
     /* When writing to the command-prompt in Windows, it is necessary
     ** to use _O_WTEXT input mode and write UTF-16 characters.
     */
@@ -119,7 +156,7 @@ int sqlite3_fputs(const char *z, FILE *out){
     if( b1==0 ) return 0;
     sz = MultiByteToWideChar(CP_UTF8, 0, z, sz, b1, sz);
     b1[sz] = 0;
-    _setmode(_fileno(out), _O_WTEXT);
+    _setmode(_fileno(out), _O_U8TEXT);
     fputws(b1, out);
     sqlite3_free(b1);
     return 0;
@@ -136,7 +173,7 @@ int sqlite3_fputs(const char *z, FILE *out){
 */
 int sqlite3_fprintf(FILE *out, const char *zFormat, ...){
   int rc;
-  if( _isatty(fileno(out)) ){
+  if( UseWtextForOutput(out) ){
     /* When writing to the command-prompt in Windows, it is necessary
     ** to use _O_WTEXT input mode and write UTF-16 characters.
     */
@@ -161,12 +198,14 @@ int sqlite3_fprintf(FILE *out, const char *zFormat, ...){
 }
 
 /*
-** Set the mode for a stream.  mode argument is typically _O_BINARY or
+** Set the mode for an output stream.  mode argument is typically _O_BINARY or
 ** _O_TEXT.
 */
 void sqlite3_fsetmode(FILE *fp, int mode){
-  fflush(fp);
+  if( !UseWtextForOutput(fp) ){
-  _setmode(_fileno(fp), mode);
+    fflush(fp);
+    _setmode(_fileno(fp), mode);
+  }
 }
 
 #endif /* defined(_WIN32) */

manifest

@@ -1,5 +1,5 @@
-C Fix\sshell\stest\scases\sto\saccount\sfor\srecent\sadditions\sto\sthe\s".help"\soutput.
+C Provide\sSQLITE_U8TEXT_ONLY\sand\sSQLITE_U8TEXT_STDIO\scompile-time\soptions\nto\sthe\ssqlite3_stdio.c\smodule.
-D 2024-09-26T01:29:22.012
+D 2024-09-26T19:16:20.290
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -425,7 +425,7 @@ F ext/misc/shathree.c 1821d90a0040c9accdbe3e3527d378d30569475d758aa70f6848924c0b
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c c0aa7b80d6df45f7da59d912b38752bcac1af53a5766966160e6c5cdd397dbea
 F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634
-F ext/misc/sqlite3_stdio.c aefcfaeefdee26eae2c0d94e41d3720d86b1dc5c96e70132366ab1e204f9ef92
+F ext/misc/sqlite3_stdio.c f110e6f2dc97c67e89f941f82af7dbd221193fa44d1e3ef38a691454a2cbccda
 F ext/misc/sqlite3_stdio.h f05eaf5e0258f0573910324a789a9586fc360a57678c57a6d63cfaa2245b6176
 F ext/misc/stmt.c b090086cd6bd6281c21271d38d576eeffe662f0e6b67536352ce32bbaa438321
 F ext/misc/stmtrand.c 59cffa5d8e158943ff1ce078956d8e208e8c04e67307e8f249dece2436dcb7fc
@@ -770,7 +770,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 9750a281f7ba073b4e6da2be1a6c4071f5d841a7746c5fb3f70d6d793b6675ea
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe
-F src/shell.c.in 5a5881cbe128962b431bfd466ff408db6bf05b971db67200615be38745f0ceef
+F src/shell.c.in f82c16f090496e61d4b67500522dbbf8f1b37c77415aafc24a7c84bc9d0c8709
 F src/sqlite.h.in 77f55bd1978a04a14db211732f0a609077cf60ba4ccf9baf39988f508945419c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
@@ -2215,8 +2215,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 660ca5ce6600d897cc2b00b9d39e5d993c1c0e71ec0d5dc706246c053a163281
+P 54c22bc6165affbea0d1166a0912d19939300ccc4ae33a6b2ed8c510ab61c574
-R 3b1b1ac1efd2caefa25e8d6c9bf0ff2f
+R 2b2b12aba8675f6b1c9a607cabf2b89d
 U drh
-Z 1e00c19e9c0388401e635e4bb988cbfe
+Z a76d82025e5362f7f888a23f868f0cc8
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-54c22bc6165affbea0d1166a0912d19939300ccc4ae33a6b2ed8c510ab61c574
+f31588520e3f45b50dcaa9eecab17f52ebb56bb53d0f9bdb88cc596d1a156353

src/shell.c.in

@@ -1631,7 +1631,13 @@ static const char *modeDescr[] = {
 #define SEP_Tab       "\t"
 #define SEP_Space     " "
 #define SEP_Comma     ","
-#define SEP_CrLf      "\r\n"
+#ifdef SQLITE_U8TEXT_ONLY
+  /* With the SQLITE_U8TEXT_ONLY option, the output will always be in
+  ** text mode.  The \r will be inserted automatically. */
+# define SEP_CrLf      "\n"
+#else
+# define SEP_CrLf      "\r\n"
+#endif
 #define SEP_Unit      "\x1F"
 #define SEP_Record    "\x1E"
 
@@ -4933,9 +4939,10 @@ static const char *(azHelp[]) = {
   ".clone NEWDB             Clone data into NEWDB from the existing database",
 #endif
   ".connection [close] [#]  Open or close an auxiliary database connection",
-#if defined(_WIN32) || defined(WIN32)
+#if defined(_WIN32) && !defined(SQLITE_U8TEXT_ONLY) \
+                    && !defined(SQLITE_U8TEXT_STDIO)
   ".crnl on|off             Translate \\n to \\r\\n.  Default ON",
-#endif
+#endif /* _WIN32 && U8TEXT_ONLY && U8TEXT_STDIO */
   ".databases               List names and files of attached databases",
   ".dbconfig ?op? ?val?     List or change sqlite3_db_config() options",
 #if SQLITE_SHELL_HAVE_RECOVER
@@ -8425,17 +8432,8 @@ static int do_meta_command(char *zLine, ShellState *p){
 
   /* Undocumented.  Legacy only.  See "crnl" below */
   if( c=='b' && n>=3 && cli_strncmp(azArg[0], "binary", n)==0 ){
-    if( nArg==2 ){
+    eputz("The \".binary\" command is deprecated. Use \".crnl\" instead.\n");
-      if( booleanValue(azArg[1]) ){
+    rc = 1;
-        sqlite3_fsetmode(p->out, _O_BINARY);
-      }else{
-        sqlite3_fsetmode(p->out, _O_TEXT);
-      }
-    }else{
-      eputz("The \".binary\" command is deprecated. Use \".crnl\" instead.\n"
-            "Usage: .binary on|off\n");
-      rc = 1;
-    }
   }else
 
   /* The undocumented ".breakpoint" command causes a call to the no-op
@@ -8561,6 +8559,10 @@ static int do_meta_command(char *zLine, ShellState *p){
   }else
 
   if( c=='c' && n==4 && cli_strncmp(azArg[0], "crnl", n)==0 ){
+#if !defined(_WIN32) || defined(SQLITE_U8TEXT_ONLY) \
+                     || defined(SQLITE_U8TEXT_STDIO)
+    sqlite3_fputs("The \".crnl\" command is disable in this build.\n", p->out);
+#else
     if( nArg==2 ){
       if( booleanValue(azArg[1]) ){
         sqlite3_fsetmode(p->out, _O_TEXT);
@@ -8568,12 +8570,10 @@ static int do_meta_command(char *zLine, ShellState *p){
         sqlite3_fsetmode(p->out, _O_BINARY);
       }
     }else{
-#if !defined(_WIN32) && !defined(WIN32)
-      eputz("The \".crnl\" is a no-op on non-Windows machines.\n");
-#endif
       eputz("Usage: .crnl on|off\n");
       rc = 1;
     }
+#endif
   }else
 
   if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){