MDEV-30691 Assertion `strlen(Ptr) == str_length' failed in void Binary_string::chop()

numerous bugs in JSON_DETAILED and multibyte charsets:

* String:chop() must be charset-aware and not simply length--
* String::append(char) must be charset-aware and not simply length++
* json_nice() first removes value_len bytes, then a
  certain number of characters

mysql-test/main/ctype_ucs.test

@@ -118,6 +118,7 @@ SET NAMES koi8r;
 SET character_set_connection=ucs2;
 --source include/ctype_like.inc
 
+--disable_service_connection
 CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET ucs2);
 INSERT INTO t1 VALUES ('<27><><EFBFBD><EFBFBD>'),('<27><><EFBFBD><EFBFBD>'),('<27><><EFBFBD><EFBFBD>'),('<27><><EFBFBD><EFBFBD>'),('<27><><EFBFBD><EFBFBD>'),('<27><><EFBFBD><EFBFBD>');
 INSERT INTO t1 VALUES ('<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>'),('<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>'),('<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>'),('<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>');
@@ -127,6 +128,7 @@ SELECT * FROM t1 WHERE a LIKE '%
 SELECT * FROM t1 WHERE a LIKE '%<25><><EFBFBD>%';
 SELECT * FROM t1 WHERE a LIKE '<27><><EFBFBD><EFBFBD>%';
 SELECT * FROM t1 WHERE a LIKE '<27><><EFBFBD><EFBFBD>%' COLLATE ucs2_bin;
+--enable_service_connection
 DROP TABLE t1;
 
 #

mysql-test/main/func_json.result

@@ -2669,3 +2669,21 @@ JSON_VALUE(@json,'$.A[last-1][last-1].key1')
 NULL
 SET @@collation_connection= @save_collation_connection;
 # End of 10.9 Test
+#
+# MDEV-30691 Assertion `strlen(Ptr) == str_length' failed in void Binary_string::chop()
+#
+set @@collation_connection=utf32_czech_ci;
+select json_detailed('[[123],456]');
+json_detailed('[[123],456]')
+[
+    [123],
+    456
+]
+set @@collation_connection=@save_collation_connection;
+select json_detailed('[[123],456]');
+json_detailed('[[123],456]')
+[
+    [123],
+    456
+]
+# End of 10.11 Test

mysql-test/main/func_json.test

@@ -1937,5 +1937,14 @@ SELECT JSON_VALUE(@json,'$.A[last-1][last-1].key1');
 
 SET @@collation_connection= @save_collation_connection;
 
-
 --echo # End of 10.9 Test
+
+--echo #
+--echo # MDEV-30691 Assertion `strlen(Ptr) == str_length' failed in void Binary_string::chop()
+--echo #
+set @@collation_connection=utf32_czech_ci;
+select json_detailed('[[123],456]');
+set @@collation_connection=@save_collation_connection;
+select json_detailed('[[123],456]');
+
+--echo # End of 10.11 Test

sql/item_jsonfunc.cc

@@ -422,14 +422,9 @@ handle_value:
       if (mode == Item_func_json_format::DETAILED && 
           value_size == 1 && je->state != JST_OBJ_END)
       {
-        for (auto i = 0; i < value_len; i++)
+        nice_js->length(nice_js->length() - value_len);
-        {
-          nice_js->chop();
-        }
         for (auto i = 0; i < (depth + 1) * tab_size + 1; i++)
-        {
           nice_js->chop();
-        }
         nice_js->append(curr_str);
       }
       

sql/sql_string.h

@@ -374,37 +374,6 @@ public:
            !memcmp(ptr(), other->ptr(), length());
   }
 
-  /*
-    PMG 2004.11.12
-    This is a method that works the same as perl's "chop". It simply
-    drops the last character of a string. This is useful in the case
-    of the federated storage handler where I'm building a unknown
-    number, list of values and fields to be used in a sql insert
-    statement to be run on the remote server, and have a comma after each.
-    When the list is complete, I "chop" off the trailing comma
-
-    ex.
-      String stringobj;
-      stringobj.append("VALUES ('foo', 'fi', 'fo',");
-      stringobj.chop();
-      stringobj.append(")");
-
-    In this case, the value of string was:
-
-    VALUES ('foo', 'fi', 'fo',
-    VALUES ('foo', 'fi', 'fo'
-    VALUES ('foo', 'fi', 'fo')
-  */
-  inline void chop()
-  {
-    if (str_length)
-    {
-      str_length--;
-      Ptr[str_length]= '\0';
-      DBUG_ASSERT(strlen(Ptr) == str_length);
-    }
-  }
-
   // Returns offset to substring or -1
   int strstr(const Binary_string &search, uint32 offset=0) const;
   int strstr(const char *search, uint32 search_length, uint32 offset=0) const;
@@ -1039,7 +1008,7 @@ public:
   }
   inline bool append(char chr)
   {
-    return Binary_string::append_char(chr);
+    return append(&chr, 1);
   }
   bool append_hex(const char *src, uint32 srclen)
   {
@@ -1162,6 +1131,15 @@ public:
     return false;
   }
 
+  inline void chop()
+  {
+    if (str_length)
+    {
+      str_length--;
+      str_length= well_formed_length();
+    }
+  }
+
   void strip_sp();
   friend String *copy_if_not_alloced(String *a,String *b,uint32 arg_length);
   friend class Field;