Only use unistr() in columnar formats when strictly needed.

Do not use unistr() in insert mode when --escape is off.
More test cases.

FossilOrigin-Name: e029828de91b10b4c7f4a19bc70c35e4f36fae4ebf32b40553a6ba9f2b3af295

manifest

@@ -1,5 +1,5 @@
-C Add\sthe\sunistr_quote()\sfunction\sthat\sworks\slike\squote(),\sbut\salso\sescape\ncontrol\scharacters\susing\sunistr()\sif\snecessary.
+C Only\suse\sunistr()\sin\scolumnar\sformats\swhen\sstrictly\sneeded.\nDo\snot\suse\sunistr()\sin\sinsert\smode\swhen\s--escape\sis\soff.\nMore\stest\scases.
-D 2025-02-24T13:51:24.322
+D 2025-02-24T17:50:49.386
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d
@@ -782,7 +782,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 626c24b258b111f75c22107aa5614ad89810df3026f5ca071116d3fe75925c75
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c a076f7db3a0fcbd9f710d7746cfc07e0b3baadee45eb3136bedc29c598ef8f1c
-F src/shell.c.in b6b9db1a32ac2befb2547d1cc6d35aacffdce20832a9ef19bc2cbd6d9c210365
+F src/shell.c.in abf1904193f1c7b6b95e2674606bd00f0b28a366b39e02a3e1e21a56ccf338d6
 F src/sqlite.h.in 8d4486fb28a90de818ac1e8c6206ea458e7de6bd8e0dfa3d554494f155be8c01
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
@@ -1647,7 +1647,7 @@ F test/shell6.test e3b883b61d4916b6906678a35f9d19054861123ad91b856461e0a456273bd
 F test/shell7.test 43fd8e511c533bab5232e95c7b4be93b243451709e89582600d4b6e67693d5c3
 F test/shell8.test aea51ecbcd4494c746b096aeff51d841d04d5f0dc4b62eb42427f16109b87acd
 F test/shell9.test 8742a5b390cdcef6369f5aa223e415aa4255a4129ef249b177887dc635a87209
-F test/shellA.test e7ff53be62d79ade789ae1d3108109ece789af43c20e4340fb1a6155c9283b09
+F test/shellA.test 78efeadb4191240914e787325dc2edab44c586c4d9d3a44a2a4ab066190e7e63
 F test/shmlock.test 3dbf017d34ab0c60abe6a44e447d3552154bd0c87b41eaf5ceacd408dd13fda5
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
@@ -2211,8 +2211,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c809997792602a7299b8ab84d018a03d291695e308ce750fc8b9d7a824edfd6e
+P e99e37b54baf7283588ead4983e613a1e14c58a0b92be5f7b25b4d9d287b5324
-R 58d2163ff5534cc6844d2cf453051f3d
+R 7e1ba0b26b7950463261b2769b28f47d
 U drh
-Z b0d6064b39fceb84b42166ec949dbfe8
+Z 01fb520867e568bd6570d54d3dd4c9c9
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-e99e37b54baf7283588ead4983e613a1e14c58a0b92be5f7b25b4d9d287b5324
+e029828de91b10b4c7f4a19bc70c35e4f36fae4ebf32b40553a6ba9f2b3af295

src/shell.c.in

@@ -4057,6 +4057,16 @@ static char *translateForDisplayAndDup(
   return (char*)zOut;
 }
 
+/* Return true if the text string z[] contains characters that need
+** unistr() escaping.
+*/
+static int needUnistr(const unsigned char *z){
+  unsigned char c;
+  if( z==0 ) return 0;
+  while( (c = *z)>0x1f || c=='\t' || c=='\n' || (c=='\r' && z[1]=='\n') ){ z++; }
+  return c!=0;
+}
+
 /* Extract the value of the i-th current column for pStmt as an SQL literal
 ** value.  Memory is obtained from sqlite3_malloc64() and must be freed by
 ** the caller.
@@ -4071,7 +4081,8 @@ static char *quoted_column(sqlite3_stmt *pStmt, int i){
       return sqlite3_mprintf("%s",sqlite3_column_text(pStmt,i));
     }
     case SQLITE_TEXT: {
-      return sqlite3_mprintf("%#Q",sqlite3_column_text(pStmt,i));
+      const unsigned char *zText = sqlite3_column_text(pStmt,i);
+      return sqlite3_mprintf(needUnistr(zText)?"%#Q":"%Q",zText);
     }
     case SQLITE_BLOB: {
       int j;
@@ -10001,6 +10012,11 @@ static int do_meta_command(char *zLine, ShellState *p){
     }else if( cli_strncmp(zMode,"insert",n2)==0 ){
       p->mode = MODE_Insert;
       set_table_name(p, zTabname ? zTabname : "table");
+      if( p->eEscMode==SHELL_ESC_OFF ){
+        ShellSetFlag(p, SHFLG_Newlines);
+      }else{
+        ShellClearFlag(p, SHFLG_Newlines);
+      }
     }else if( cli_strncmp(zMode,"quote",n2)==0 ){
       p->mode = MODE_Quote;
       sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);

test/shellA.test

@@ -63,7 +63,7 @@ do_test shellA-1.2 {
 └───┴──────────────────────────┘
 }
 
-# Default output mode uses symbols for control characters
+# ".mode list"
 #
 do_test shellA-1.3 {
   exec $CLI test.db {SELECT x FROM t1 WHERE a=2;}
@@ -71,19 +71,190 @@ do_test shellA-1.3 {
 ␛[31mVT-100 codes␛[0m
 }
 do_test shellA-1.4 {
+  exec $CLI test.db --escape symbol {SELECT x FROM t1 WHERE a=2;}
+} {
+␛[31mVT-100 codes␛[0m
+}
+do_test shellA-1.5 {
   exec $CLI test.db --escape ascii {SELECT x FROM t1 WHERE a=2;}
 } {
 ^[[31mVT-100 codes^[[0m
 }
-do_test shellA-1.5 {
+do_test shellA-1.6 {
   exec $CLI test.db {.mode list --escape symbol} {SELECT x FROM t1 WHERE a=2;}
 } {
 ␛[31mVT-100 codes␛[0m
 }
-do_test shellA-1.6 {
+do_test shellA-1.7 {
   exec $CLI test.db {.mode list --escape ascii} {SELECT x FROM t1 WHERE a=2;}
 } {
 ^[[31mVT-100 codes^[[0m
 }
+do_test shellA-1.8 {
+  file delete -force out.txt
+  exec $CLI test.db {.mode list --escape off} {SELECT x FROM t1 WHERE a=7;} \
+     >out.txt
+  set fd [open out.txt rb]
+  set res [read $fd]
+  close $fd
+  string trim $res
+} "carriage\rreturn"
+do_test shellA-1.9 {
+  set rc [catch {
+     exec $CLI test.db {.mode test --escape xyz}
+  } msg]
+  lappend rc $msg
+} {1 {unknown control character escape mode "xyz" - choices: symbol ascii off}}
+do_test shellA-1.10 {
+  set rc [catch {
+     exec $CLI --escape abc test.db .q
+  } msg]
+  lappend rc $msg
+} {1 {unknown control character escape mode "abc" - choices: symbol ascii off}}
+
+# ".mode quote"
+#
+do_test shellA-2.1 {
+ exec $CLI test.db --quote {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+1,'line with '' single quote'
+2,unistr('\u001b[31mVT-100 codes\u001b[0m')
+6,'new
+line'
+7,unistr('carriage\u000dreturn')
+8,'last line'
+}
+do_test shellA-2.2 {
+  exec $CLI test.db --quote {.mode}
+} {current output mode: quote --escape symbol}
+do_test shellA-2.3 {
+  exec $CLI test.db --quote --escape ASCII {.mode}
+} {current output mode: quote --escape ascii}
+do_test shellA-2.4 {
+  exec $CLI test.db --quote --escape OFF {.mode}
+} {current output mode: quote --escape off}
+
+
+# ".mode line"
+#
+do_test shellA-3.1 {
+ exec $CLI test.db --line --escape symbol \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+    a = 1
+    x = line with ' single quote
+
+    a = 2
+    x = ␛[31mVT-100 codes␛[0m
+
+    a = 6
+    x = new
+line
+
+    a = 7
+    x = carriage␍return
+
+    a = 8
+    x = last line
+}
+do_test shellA-3.2 {
+ exec $CLI test.db --line --escape ascii \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+    a = 1
+    x = line with ' single quote
+
+    a = 2
+    x = ^[[31mVT-100 codes^[[0m
+
+    a = 6
+    x = new
+line
+
+    a = 7
+    x = carriage^Mreturn
+
+    a = 8
+    x = last line
+}
+
+# ".mode box"
+#
+do_test shellA-4.1 {
+ exec $CLI test.db --box --escape ascii \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+┌───┬──────────────────────────┐
+│ a │            x             │
+├───┼──────────────────────────┤
+│ 1 │ line with ' single quote │
+├───┼──────────────────────────┤
+│ 2 │ ^[[31mVT-100 codes^[[0m  │
+├───┼──────────────────────────┤
+│ 6 │ new                      │
+│   │ line                     │
+├───┼──────────────────────────┤
+│ 7 │ carriage^Mreturn         │
+├───┼──────────────────────────┤
+│ 8 │ last line                │
+└───┴──────────────────────────┘
+}
+do_test shellA-4.2 {
+ exec $CLI test.db {.mode qbox} {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+┌───┬───────────────────────────────────────────┐
+│ a │                     x                     │
+├───┼───────────────────────────────────────────┤
+│ 1 │ 'line with '' single quote'               │
+├───┼───────────────────────────────────────────┤
+│ 2 │ unistr('\u001b[31mVT-100 codes\u001b[0m') │
+├───┼───────────────────────────────────────────┤
+│ 6 │ 'new                                      │
+│   │ line'                                     │
+├───┼───────────────────────────────────────────┤
+│ 7 │ unistr('carriage\u000dreturn')            │
+├───┼───────────────────────────────────────────┤
+│ 8 │ 'last line'                               │
+└───┴───────────────────────────────────────────┘
+}
+
+# ".mode insert"
+#
+do_test shellA-5.1 {
+ exec $CLI test.db {.mode insert t1 --escape ascii} \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+INSERT INTO t1 VALUES(1,'line with '' single quote');
+INSERT INTO t1 VALUES(2,unistr('\u001b[31mVT-100 codes\u001b[0m'));
+INSERT INTO t1 VALUES(6,unistr('new\u000aline'));
+INSERT INTO t1 VALUES(7,unistr('carriage\u000dreturn'));
+INSERT INTO t1 VALUES(8,'last line');
+}
+do_test shellA-5.2 {
+ exec $CLI test.db {.mode insert t1 --escape symbol} \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+INSERT INTO t1 VALUES(1,'line with '' single quote');
+INSERT INTO t1 VALUES(2,unistr('\u001b[31mVT-100 codes\u001b[0m'));
+INSERT INTO t1 VALUES(6,unistr('new\u000aline'));
+INSERT INTO t1 VALUES(7,unistr('carriage\u000dreturn'));
+INSERT INTO t1 VALUES(8,'last line');
+}
+do_test shellA-5.3 {
+  file delete -force out.txt
+  exec $CLI test.db {.mode insert t1 --escape off} \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} >out.txt
+  set fd [open out.txt rb]
+  set res [read $fd]
+  close $fd
+  string trim [string map [list \r\n \n] $res]
+} "
+INSERT INTO t1 VALUES(1,'line with '' single quote');
+INSERT INTO t1 VALUES(2,'\033\13331mVT-100 codes\033\1330m');
+INSERT INTO t1 VALUES(6,'new
+line');
+INSERT INTO t1 VALUES(7,'carriage\rreturn');
+INSERT INTO t1 VALUES(8,'last line');
+"
 
 finish_test