From 0b453b3b3379f34527c957c6e1f7631a887c7340 Mon Sep 17 00:00:00 2001
From: drh <>
Date: Fri, 11 Oct 2024 14:02:48 +0000
Subject: [PATCH] Avoid undesirable NL to CRLF translation when doing binary
 output to the Windows console.

FossilOrigin-Name: d25bdce36abed95524ad058a277aba7bb17270e7ff1476474713dbc29742c762
---
 ext/misc/sqlite3_stdio.c | 84 ++++++++++++++++++++++++++++++++++++----
 manifest                 | 15 ++++---
 manifest.uuid            |  2 +-
 src/shell.c.in           | 17 ++++----
 4 files changed, 92 insertions(+), 26 deletions(-)

diff --git a/ext/misc/sqlite3_stdio.c b/ext/misc/sqlite3_stdio.c
index 46f12dff36..393815522b 100644
--- a/ext/misc/sqlite3_stdio.c
+++ b/ext/misc/sqlite3_stdio.c
@@ -67,6 +67,31 @@
 # define IsConsole(fd)         1
 #endif
 
+/*
+** Global variables determine if simulated O_BINARY mode is to be
+** used for stdout or other, respectively.  Simulated O_BINARY mode
+** means the mode is usually O_BINARY, but switches to O_U8TEXT for
+** unicode characters U+0080 or greater (any character that has a
+** multi-byte representation in UTF-8).  This is the only way we
+** have found to render Unicode characters on a Windows console while
+** at the same time avoiding undesirable \n to \r\n translation.
+*/
+static int simBinaryStdout = 0;
+static int simBinaryOther = 0;
+
+
+/*
+** Determine if simulated binary mode should be used for output to fd
+*/
+static int UseBinaryWText(FILE *fd){
+  if( fd==stdout || fd==stderr ){
+    return simBinaryStdout;
+  }else{
+    return simBinaryOther;
+  }
+}
+
+
 /*
 ** Work-alike for the fopen() routine from the standard C library.
 */
@@ -88,6 +113,7 @@ FILE *sqlite3_fopen(const char *zFilename, const char *zMode){
   }
   free(b1);
   free(b2);
+  simBinaryOther = 0;
   return fp;
 }
 
@@ -143,13 +169,53 @@ char *sqlite3_fgets(char *buf, int sz, FILE *in){
   }
 }
 
+/*
+** Send ASCII text as O_BINARY.  But for Unicode characters U+0080 and
+** greater, switch to O_U8TEXT.
+*/
+static void piecemealOutput(wchar_t *b1, int sz, FILE *out){
+  int i;
+  wchar_t c;
+  while( sz>0 ){
+    for(i=0; i<sz && b1[i]>=0x80; i++){}
+    if( i>0 ){
+      c = b1[i];
+      b1[i] = 0;
+      fflush(out);
+      _setmode(_fileno(out), _O_U8TEXT);
+      fputws(b1, out);
+      fflush(out);
+      b1 += i;
+      b1[0] = c;
+      sz -= i;
+    }else{
+      fflush(out);
+      _setmode(_fileno(out), _O_TEXT);
+      _setmode(_fileno(out), _O_BINARY);
+      fwrite(&b1[0], 1, 1, out);
+      for(i=1; i<sz && b1[i]<0x80; i++){
+        fwrite(&b1[i], 1, 1, out);
+      }
+      fflush(out);
+      _setmode(_fileno(out), _O_U8TEXT);
+      b1 += i;
+      sz -= i;
+    }
+  }
+}
+
 /*
 ** Work-alike for fputs() from the standard C library.
 */
 int sqlite3_fputs(const char *z, FILE *out){
-  if( UseWtextForOutput(out) ){
+  if( !UseWtextForOutput(out) ){
+    /* Writing to a file or other destination, just write bytes without
+    ** any translation. */
+    return fputs(z, out);
+  }else{
     /* When writing to the command-prompt in Windows, it is necessary
-    ** to use _O_WTEXT input mode and write UTF-16 characters.
+    ** to use O_U8TEXT to render Unicode U+0080 and greater.  Go ahead
+    ** use O_U8TEXT for everything in text mode.
     */
     int sz = (int)strlen(z);
     wchar_t *b1 = malloc( (sz+1)*sizeof(wchar_t) );
@@ -157,13 +223,13 @@ int sqlite3_fputs(const char *z, FILE *out){
     sz = MultiByteToWideChar(CP_UTF8, 0, z, sz, b1, sz);
     b1[sz] = 0;
     _setmode(_fileno(out), _O_U8TEXT);
-    fputws(b1, out);
+    if( UseBinaryWText(out) ){
+      piecemealOutput(b1, sz, out);
+    }else{
+      fputws(b1, out);
+    }
     sqlite3_free(b1);
     return 0;
-  }else{
-    /* Writing to a file or other destination, just write bytes without
-    ** any translation. */
-    return fputs(z, out);
   }
 }
 
@@ -205,6 +271,10 @@ void sqlite3_fsetmode(FILE *fp, int mode){
   if( !UseWtextForOutput(fp) ){
     fflush(fp);
     _setmode(_fileno(fp), mode);
+  }else if( fp==stdout || fp==stderr ){
+    simBinaryStdout = (mode==_O_BINARY);
+  }else{
+    simBinaryOther = (mode==_O_BINARY);
   }
 }
 
diff --git a/manifest b/manifest
index 82bf253abc..f410abbe68 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improvements\sto\sTCL9\ssupport.\s\sFixes\sto\sthe\sMakefiles\sso\sthat\sthe\s"install"\ntargets\sare\sconsistent\sand\sso\sthat\sthey\swork\sbetter\swith\sTCL9.
-D 2024-10-10T10:47:37.943
+C Avoid\sundesirable\sNL\sto\sCRLF\stranslation\swhen\sdoing\sbinary\soutput\sto\sthe\nWindows\sconsole.
+D 2024-10-11T14:02:48.369
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -427,7 +427,7 @@ F ext/misc/shathree.c 1821d90a0040c9accdbe3e3527d378d30569475d758aa70f6848924c0b
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c bcc42ef3fd29429bc01a83e751332b8d4690e65d45008449bdffe7656371487f
 F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634
-F ext/misc/sqlite3_stdio.c f110e6f2dc97c67e89f941f82af7dbd221193fa44d1e3ef38a691454a2cbccda
+F ext/misc/sqlite3_stdio.c 74d86044cc45475cb0f7fa0d07e7bb5b3cc77c4339b084af7a4afe1b9cc008ef
 F ext/misc/sqlite3_stdio.h f05eaf5e0258f0573910324a789a9586fc360a57678c57a6d63cfaa2245b6176
 F ext/misc/stmt.c b090086cd6bd6281c21271d38d576eeffe662f0e6b67536352ce32bbaa438321
 F ext/misc/stmtrand.c 59cffa5d8e158943ff1ce078956d8e208e8c04e67307e8f249dece2436dcb7fc
@@ -770,7 +770,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c c8a5372b97b2a2e972a280676f06ddb5b74e885d3b1f5ce383f839907b57ef68
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe
-F src/shell.c.in 1b5fda8bebb23783985744fde738976791fca5fd98d4382c4572e13cce2a0e29
+F src/shell.c.in 0701ac81f16d1b23254f141118ea064c33c153d7099d3bf7db650ce45eead689
 F src/sqlite.h.in 1def838497ad53c81486649ce79821925d1ac20a9843af317a344d507efe116e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
@@ -2216,9 +2216,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 41d58a014ce89356932d717843a1fa6e0735f15a7b7265c41ac85a9722a5d826 69346e9d4704e8fd82cbb3359913191e05cb38ee591baf97dcfe321db0ea085e
-R cb2da8047bc68c186c40f7fac828ca27
-T +closed 69346e9d4704e8fd82cbb3359913191e05cb38ee591baf97dcfe321db0ea085e
+P bcfae7183e92ce37717852bae5b1dd526903fa8429fb6f738c2147d4e5231642
+R ce21189a126f3d2c64453710bc34a276
 U drh
-Z c26071f7bb2f67975d197a849e9477c9
+Z 1e590b34c76a965a6a2ae1e5a7b7871c
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 3ab3edf13f..e116a9db35 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bcfae7183e92ce37717852bae5b1dd526903fa8429fb6f738c2147d4e5231642
+d25bdce36abed95524ad058a277aba7bb17270e7ff1476474713dbc29742c762
diff --git a/src/shell.c.in b/src/shell.c.in
index 50eb3ae2c0..5a90fec7e0 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -5953,7 +5953,7 @@ static void output_file_close(FILE *f){
 ** recognized and do the right thing.  NULL is returned if the output
 ** filename is "off".
 */
-static FILE *output_file_open(const char *zFile, int bTextMode){
+static FILE *output_file_open(const char *zFile){
   FILE *f;
   if( cli_strcmp(zFile,"stdout")==0 ){
     f = stdout;
@@ -5962,7 +5962,7 @@ static FILE *output_file_open(const char *zFile, int bTextMode){
   }else if( cli_strcmp(zFile, "off")==0 ){
     f = 0;
   }else{
-    f = sqlite3_fopen(zFile, bTextMode ? "w" : "wb");
+    f = sqlite3_fopen(zFile, "w");
     if( f==0 ){
       sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile);
     }
@@ -9663,7 +9663,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
       output_file_close(p->pLog);
       if( cli_strcmp(zFile,"on")==0 ) zFile = "stdout";
-      p->pLog = output_file_open(zFile, 0);
+      p->pLog = output_file_open(zFile);
     }
   }else
 
@@ -9918,7 +9918,6 @@ static int do_meta_command(char *zLine, ShellState *p){
    || (c=='w' && n==3 && cli_strcmp(azArg[0],"www")==0)
   ){
     char *zFile = 0;
-    int bTxtMode = 0;
     int i;
     int eMode = 0;
     int bOnce = 0;            /* 0: .output, 1: .once, 2: .excel/.www */
@@ -10000,11 +9999,9 @@ static int do_meta_command(char *zLine, ShellState *p){
         /* web-browser mode. */
         newTempFile(p, "html");
         if( !bPlain ) p->mode = MODE_Www;
-        bTxtMode = 1;
       }else{
         /* text editor mode */
         newTempFile(p, "txt");
-        bTxtMode = 1;
       }
       sqlite3_free(zFile);
       zFile = sqlite3_mprintf("%s", p->zTempFile);
@@ -10028,7 +10025,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
 #endif
     }else{
-      FILE *pfFile = output_file_open(zFile, bTxtMode);
+      FILE *pfFile = output_file_open(zFile);
       if( pfFile==0 ){
         if( cli_strcmp(zFile,"off")!=0 ){
           sqlite3_fprintf(stderr,"Error: cannot write to \"%s\"\n", zFile);
@@ -10036,13 +10033,13 @@ static int do_meta_command(char *zLine, ShellState *p){
         rc = 1;
       } else {
         output_redir(p, pfFile);
+        if( zBom ) sqlite3_fputs(zBom, pfFile);
         if( bPlain && eMode=='w' ){
           sqlite3_fputs(
             "<!DOCTYPE html>\n<BODY>\n<PLAINTEXT>\n",
             pfFile
           );
         }
-        if( zBom ) sqlite3_fputs(zBom, pfFile);
         sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile);
       }
     }
@@ -11225,7 +11222,7 @@ static int do_meta_command(char *zLine, ShellState *p){
   /* Begin redirecting output to the file "testcase-out.txt" */
   if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){
     output_reset(p);
-    p->out = output_file_open("testcase-out.txt", 0);
+    p->out = output_file_open("testcase-out.txt");
     if( p->out==0 ){
       eputz("Error: cannot open 'testcase-out.txt'\n");
     }
@@ -11729,7 +11726,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         }
       }else{
         output_file_close(p->traceOut);
-        p->traceOut = output_file_open(z, 0);
+        p->traceOut = output_file_open(z);
       }
     }
     if( p->traceOut==0 ){