Ensure that source files contain only valid UTF8 encodings

Modern software (including text editors, static analysis software,
and web-based code review interfaces) often requires source code files
to be interpretable via a consistent character encoding, with UTF-8 or
ASCII (a strict subset of UTF-8) as the default. Several of the MariaDB
source files contain bytes that are not valid in either the UTF-8 or
ASCII encodings, but instead represent strings encoded in the
ISO-8859-1/Latin-1 or ISO-8859-2/Latin-2 encodings.

These inconsistent encodings may prevent software from correctly
presenting or processing such files. Converting all source files to
valid UTF8 characters will ensure correct handling.

Comments written in Czech were replaced with lightly-corrected
translations from Google Translate. Additionally, comments describing
the proper handling of special characters were changed so that the
comments are now purely UTF8.

All new code of the whole pull request, including one or several files
that are either new files or modified ones, are contributed under the
BSD-new license. I am contributing on behalf of my employer
Amazon Web Services, Inc.

mysys/my_win_popen.cc

@@ -92,7 +92,7 @@ extern "C" FILE *my_win_popen(const char *cmd, const char *mode)
       goto error;
     break;
   default:
-    /* Unknown mode, <EFBFBD>xpected "r", "rt", "w", "wt" */
+    /* Unknown mode, expected "r", "rt", "w", "wt" */
     abort();
   }
   if (!SetHandleInformation(parent_pipe_end, HANDLE_FLAG_INHERIT, 0))

storage/connect/domdoc.cpp

@@ -642,7 +642,6 @@ bool DOMNODELIST::DropItem(PGLOBAL g, int n)
 	if (Listp == NULL || Listp->length < n)
 		return true;
 
-//Listp->item[n] = NULL;  La propri<72>t<EFBFBD> n'a pas de m<>thode 'set'
   return false;
 }  // end of DeleteItem
 

strings/ctype-czech.c

@@ -23,13 +23,13 @@
 	solution was needed than the one-to-one conversion table. To
 	note a few, here is an example of a Czech sorting sequence:
 
-		co < hlaska < hl<EFBFBD>ska < hlava < chlapec < krtek
+		co < hlaska < hláska < hlava < chlapec < krtek
 
 	It because some of the rules are: double char 'ch' is sorted
-	between 'h' and 'i'. Accented character '<EFBFBD>' (a with acute) is
+	between 'h' and 'i'. Accented character 'á' (a with acute) is
 	sorted after 'a' and before 'b', but only if the word is
 	otherwise the same. However, because 's' is sorted before 'v'
-	in hlava, the accentness of '<EFBFBD>' is overridden. There are many
+	in hlava, the accentness of 'á' is overridden. There are many
 	more rules.
 
 	This file defines functions my_strxfrm and my_strcoll for
@@ -42,8 +42,9 @@
 	passes, that's why we need four times more space for expanded
 	string.
 
-	This file also contains the ISO-Latin-2 definitions of
-	characters.
+	The non-ASCII literal strings in this file are encoded
+	in the iso-8859-2 / latin-2 character set
+	(https://en.wikipedia.org/wiki/ISO/IEC_8859-2)
 
 	Author: (c) 1997--1998 Jan Pazdziora, adelton@fi.muni.cz
 	Jan Pazdziora has a shared copyright for this code
@@ -112,7 +113,7 @@ static const struct wordvalue doubles[] = {
 	};
 
 /*
-	Unformal description of the algorithm:
+	Informal description of the algorithm:
 
 	We walk the string left to right.
 
@@ -127,7 +128,7 @@ static const struct wordvalue doubles[] = {
 
 	End of pass is marked with value 1 on the output.
 
-	For each character, we read it's value from the table.
+	For each character, we read its value from the table.
 
 	If the value is ignore (0), we go straight to the next character.
 
@@ -139,31 +140,6 @@ static const struct wordvalue doubles[] = {
 	exists behind it, find its value.
 
 	We append 0 to the end.
----
-	Neform<72>ln<6C> popis algoritmu:
-
-	Proch<63>z<EFBFBD>me <20>et<65>zec zleva doprava.
-
-	Konec <20>et<65>zce je p<>ed<65>n bu<62> jako parametr, nebo je to *p == 0.
-	Toto je o<>et<65>eno makrem IS_END.
-
-	Pokud jsme do<64>li na konec <20>et<65>zce p<>i pr<70>chodu 0, nejdeme na
-	za<7A><61>tek, ale na ulo<6C>enou pozici, proto<74>e prvn<76> a druh<75> pr<70>chod
-	b<><62><EFBFBD> sou<6F>asn<73>.
-
-	Konec vstupu (pr<70>chodu) ozna<6E><61>me na v<>stupu hodnotou 1.
-
-	Pro ka<6B>d<EFBFBD> znak <20>et<65>zce na<6E>teme hodnotu z t<><74>d<EFBFBD>c<EFBFBD> tabulky.
-
-	Jde-li o hodnotu ignorovat (0), sko<6B><6F>me ihned na dal<61><6C> znak..
-
-	Jde-li o hodnotu konec slova (2) a je to pr<70>chod 0 nebo 1,
-	p<>esko<6B><6F>me v<>echny dal<61><6C> 0 -- 2 a prohod<6F>me pr<70>chody.
-
-	Jde-li o kompozitn<74> znak (255), otestujeme, zda n<>sleduje
-	spr<70>vn<76> do dvojice, dohled<65>me spr<70>vnou hodnotu.
-
-	Na konci p<>ipoj<6F>me znak 0
  */
 
 #define ADD_TO_RESULT(dest, len, totlen, value)			\
@@ -336,24 +312,23 @@ my_strnxfrm_czech(CHARSET_INFO *cs __attribute__((unused)),
 
 
 /*
-	Neform<EFBFBD>ln<EFBFBD> popis algoritmu:
+	Informal description of the algorithm:
 
-	proch<EFBFBD>z<EFBFBD>me <20>et<65>zec zleva doprava
-	konec <20>et<65>zce pozn<7A>me podle *p == 0
-	pokud jsme do<64>li na konec <20>et<65>zce p<>i pr<70>chodu 0, nejdeme na
-		za<EFBFBD><EFBFBD>tek, ale na ulo<6C>enou pozici, proto<74>e prvn<76> a druh<75>
-		pr<EFBFBD>chod b<><62><EFBFBD> sou<6F>asn<73>
-	konec vstupu (pr<EFBFBD>chodu) ozna<6E><61>me na v<>stupu hodnotou 1
+	we pass the chain from left to right
+	we know the end of the string by *p == 0
+	if we reached the end of the string on transition 0, then we don't go to
+		start, but to the saved position, because the first and second
+		the passage runs concurrently
+	we mark the end of the input (transition) with the value 1 on the output
 
-	na<EFBFBD>teme hodnotu z t<><74>d<EFBFBD>c<EFBFBD> tabulky
-	jde-li o hodnotu ignorovat (0), sko<EFBFBD><EFBFBD>me na dal<61><6C> pr<70>chod
-	jde-li o hodnotu konec slova (2) a je to pr<70>chod 0 nebo 1,
-		p<EFBFBD>esko<EFBFBD><EFBFBD>me v<>echny dal<61><6C> 0 -- 2 a prohod<6F>me
-		pr<70>chody
-	jde-li o kompozitn<74> znak (255), otestujeme, zda n<>sleduje
-		spr<70>vn<76> do dvojice, dohled<65>me spr<70>vnou hodnotu
+	then we load the value from the sorting table
+	if the value is ignore (0), we jump to the next pass
+	if the value is the end of the word (2) and it is a 0 or 1 transition,
+		we skip all the other 0 -- 2 and switch transitions
+	if it is a composite character (255), we test whether it follows
+		correct to the pair, we find the correct value
 
-	na konci p<>ipoj<6F>me znak 0
+	then we add the character 0 at the end
  */
 
 

strings/ctype-latin1.c

@@ -504,19 +504,19 @@ struct charset_info_st my_charset_latin1_nopad=
  *
  * The modern sort order is used, where:
  *
- * '<EFBFBD>'  ->  "ae"
- * '<EFBFBD>'  ->  "oe"
- * '<EFBFBD>'  ->  "ue"
- * '<EFBFBD>'  ->  "ss"
+ * 'ä'  ->  "ae"
+ * 'ö'  ->  "oe"
+ * 'ü'  ->  "ue"
+ * 'ß'  ->  "ss"
  */
 
 
 /*
  * This is a simple latin1 mapping table, which maps all accented
  * characters to their non-accented equivalents.  Note: in this
- * table, '<EFBFBD>' is mapped to 'A', '<EFBFBD>' is mapped to 'Y', etc. - all
+ * table, 'ä' is mapped to 'A', 'ÿ' is mapped to 'Y', etc. - all
  * accented characters except the following are treated the same way.
- * <EFBFBD>, <EFBFBD>, <EFBFBD>, <EFBFBD>, <EFBFBD>, <EFBFBD>
+ * Ü, ü, Ö, ö, Ä, ä
  */
 
 static const uchar sort_order_latin1_de[] = {
@@ -582,7 +582,7 @@ static const uchar combo2map[]={
   my_strnxfrm_latin_de() on both strings and compared the result strings.
 
   This means that:
-  <EFBFBD> must also matches <EFBFBD>E and A<EFBFBD>, because my_strxn_frm_latin_de() will convert
+  Ä must also matches ÁE and Aè, because my_strxn_frm_latin_de() will convert
   both to AE.
 
   The other option would be to not do any accent removal in
@@ -708,7 +708,7 @@ void my_hash_sort_latin1_de(CHARSET_INFO *cs __attribute__((unused)),
     
   /*
     Remove end space. We have to do this to be able to compare
-    'AE' and '<EFBFBD>' as identical
+    'AE' and 'Ä' as identical
   */
   end= skip_trailing_space(key, len);