hack to speed up UTF-8 processing

2006-08-03  Paolo Bonzini  <bonzini@gnu.org>

	* sed/execute.c (str_append): Use is_utf8 to skip useless work.
	* sed/mbcs.c (initialize_mbcs): Look for a UTF-8 locale.
	(is_utf8): New.
	* sed/sed.h (is_utf8): New.

git-archimport-id: bonzini@gnu.org--2004b/sed--stable--4.1--patch-70
This commit is contained in:
Paolo Bonzini committed 2008-01-09 16:12:10 +01:00
1 parent 4ca34ac568
commit 3ca529fbe2
4 files changed
+51 -17

No files matched your search

+7
View File
@@ -1,3 +1,10 @@
2006-08-03 Paolo Bonzini <bonzini@gnu.org>
* sed/execute.c (str_append): Use is_utf8 to skip useless work.
* sed/mbcs.c (initialize_mbcs): Look for a UTF-8 locale.
(is_utf8): New.
* sed/sed.h (is_utf8): New.
2006-07-13 Paolo Bonzini <bonzini@gnu.org>
* bootstrap.sh.in: Add a few autoconfy tests.
+18 -17
View File
@@ -235,25 +235,26 @@ str_append(to, string, length)
to->length = new_length;
#ifdef HAVE_MBRTOWC
if (mb_cur_max == 1)
return;
if (mb_cur_max > 1 && !is_utf8)
while (length)
{
size_t n = MBRLEN (string, length, &to->mbstate);
while (length)
{
int n = MBRLEN (string, length, &to->mbstate);
/* An invalid sequence is treated like a singlebyte character. */
if (n == (size_t) -1)
{
memset (&to->mbstate, 0, sizeof (to->mbstate));
n = 1;
}
/* An invalid sequence is treated like a singlebyte character. */
if (n == -1)
{
memset (&to->mbstate, 0, sizeof (to->mbstate));
n = 1;
}
if (n > 0)
length -= n;
else
break;
}
if (n > 0)
{
string += n;
length -= n;
}
else
break;
}
#endif
}
+25
View File
@@ -18,7 +18,12 @@
#include "sed.h"
#include <stdlib.h>
#ifdef HAVE_LANGINFO_CODESET
#include <langinfo.h>
#endif
int mb_cur_max;
bool is_utf8;
#ifdef HAVE_MBRTOWC
/* Add a byte to the multibyte character represented by the state
@@ -47,6 +52,26 @@ int brlen (ch, cur_stat)
void
initialize_mbcs ()
{
/* For UTF-8, we know that the encoding is stateless. */
const char *codeset_name;
#ifdef HAVE_LANGINFO_CODESET
codeset_name = nl_langinfo (CODESET);
#else
codeset_name = getenv ("LC_ALL");
if (codeset_name == NULL || codeset_name[0] == '\0')
codeset_name = getenv ("LC_CTYPE");
if (codeset_name == NULL || codeset_name[0] == '\0')
codeset_name = getenv ("LANG");
if (codeset_name == NULL)
codeset_name = "";
else if (strchr (codeset_name, '.') != NULL)
codeset_name = strchr (codeset_name, '.') + 1;
#endif
is_utf8 = (strcasecmp (codeset_name, "UTF-8") == 0
|| strcasecmp (codeset_name, "UTF8") == 0);
#ifdef HAVE_MBRTOWC
mb_cur_max = MB_CUR_MAX;
#else
+1
View File
@@ -233,6 +233,7 @@ extern bool use_extended_syntax_p;
/* Declarations for multibyte character sets. */
extern int mb_cur_max;
extern bool is_utf8;
#ifdef HAVE_MBRTOWC
#ifdef HAVE_BTOWC