Remove ts_locale.c's t_isdigit(), t_isspace(), t_isprint()

These do the same thing as the standard isdigit(), isspace(), and
isprint() but with multibyte and encoding support.  But all the
callers are only interested in analyzing single-byte ASCII characters.
So this extra layer is overkill and we can replace the uses with the
standard functions.

All the t_is*() functions in ts_locale.c are under scrutiny because
they don't use the common locale provider framework but instead use
the global libc locale settings.  For the functions being touched by
this patch, we don't need all that anyway, as mentioned above, so the
simplest solution is to just remove them.  The few remaining t_is*()
functions will need a different treatment in a separate patch.

pg_trgm has some compile-time options with macros such as
KEEPONLYALNUM.  These are not documented, and the non-default variant
is not supported by any test cases.  As part of this undertaking, I'm
removing the non-default variant, as it is in the way of cleanup.  So
in this case, the not-KEEPONLYALNUM code path is gone.

Reviewed-by: Jeff Davis <pgsql@j-davis.com>
Discussion: https://www.postgresql.org/message-id/flat/653f3b84-fc87-45a7-9a0c-bfb4fcab3e7d%40eisentraut.org

contrib/dict_xsyn/dict_xsyn.c

@@ -48,14 +48,14 @@ find_word(char *in, char **end)
 	char	   *start;
 
 	*end = NULL;
-	while (*in && t_isspace(in))
+	while (*in && isspace((unsigned char) *in))
 		in += pg_mblen(in);
 
 	if (!*in || *in == '#')
 		return NULL;
 	start = in;
 
-	while (*in && !t_isspace(in))
+	while (*in && !isspace((unsigned char) *in))
 		in += pg_mblen(in);
 
 	*end = in;

contrib/ltree/ltree_io.c

@@ -411,7 +411,7 @@ parse_lquery(const char *buf, struct Node *escontext)
 			case LQPRS_WAITFNUM:
 				if (t_iseq(ptr, ','))
 					state = LQPRS_WAITSNUM;
-				else if (t_isdigit(ptr))
+				else if (isdigit((unsigned char) *ptr))
 				{
 					int			low = atoi(ptr);
 
@@ -429,7 +429,7 @@ parse_lquery(const char *buf, struct Node *escontext)
 					UNCHAR;
 				break;
 			case LQPRS_WAITSNUM:
-				if (t_isdigit(ptr))
+				if (isdigit((unsigned char) *ptr))
 				{
 					int			high = atoi(ptr);
 
@@ -460,7 +460,7 @@ parse_lquery(const char *buf, struct Node *escontext)
 			case LQPRS_WAITCLOSE:
 				if (t_iseq(ptr, '}'))
 					state = LQPRS_WAITEND;
-				else if (!t_isdigit(ptr))
+				else if (!isdigit((unsigned char) *ptr))
 					UNCHAR;
 				break;
 			case LQPRS_WAITND:
@@ -471,7 +471,7 @@ parse_lquery(const char *buf, struct Node *escontext)
 				}
 				else if (t_iseq(ptr, ','))
 					state = LQPRS_WAITSNUM;
-				else if (!t_isdigit(ptr))
+				else if (!isdigit((unsigned char) *ptr))
 					UNCHAR;
 				break;
 			case LQPRS_WAITEND:

contrib/ltree/ltxtquery_io.c

@@ -88,7 +88,7 @@ gettoken_query(QPRS_STATE *state, int32 *val, int32 *lenval, char **strval, uint
 					*lenval = charlen;
 					*flag = 0;
 				}
-				else if (!t_isspace(state->buf))
+				else if (!isspace((unsigned char) *state->buf))
 					ereturn(state->escontext, ERR,
 							(errcode(ERRCODE_SYNTAX_ERROR),
 							 errmsg("operand syntax error")));

contrib/pg_trgm/trgm.h

@@ -15,7 +15,6 @@
  */
 #define LPADDING		2
 #define RPADDING		1
-#define KEEPONLYALNUM
 /*
  * Caution: IGNORECASE macro means that trigrams are case-insensitive.
  * If this macro is disabled, the ~* and ~~* operators must be removed from
@@ -51,13 +50,8 @@ typedef char trgm[3];
 	*(((char*)(a))+2) = *(((char*)(b))+2);	\
 } while(0)
 
-#ifdef KEEPONLYALNUM
 #define ISWORDCHR(c)	(t_isalnum(c))
 #define ISPRINTABLECHAR(a)	( isascii( *(unsigned char*)(a) ) && (isalnum( *(unsigned char*)(a) ) || *(unsigned char*)(a)==' ') )
-#else
-#define ISWORDCHR(c)	(!t_isspace(c))
-#define ISPRINTABLECHAR(a)	( isascii( *(unsigned char*)(a) ) && isprint( *(unsigned char*)(a) ) )
-#endif
 #define ISPRINTABLETRGM(t)	( ISPRINTABLECHAR( ((char*)(t)) ) && ISPRINTABLECHAR( ((char*)(t))+1 ) && ISPRINTABLECHAR( ((char*)(t))+2 ) )
 
 #define ISESCAPECHAR(x) (*(x) == '\\')	/* Wildcard escape character */

contrib/unaccent/unaccent.c

@@ -155,7 +155,7 @@ initTrie(const char *filename)
 				{
 					ptrlen = pg_mblen(ptr);
 					/* ignore whitespace, but end src or trg */
-					if (t_isspace(ptr))
+					if (isspace((unsigned char) *ptr))
 					{
 						if (state == 1)
 							state = 2;

src/backend/tsearch/dict_synonym.c

@@ -47,7 +47,7 @@ findwrd(char *in, char **end, uint16 *flags)
 	char	   *lastchar;
 
 	/* Skip leading spaces */
-	while (*in && t_isspace(in))
+	while (*in && isspace((unsigned char) *in))
 		in += pg_mblen(in);
 
 	/* Return NULL on empty lines */
@@ -60,7 +60,7 @@ findwrd(char *in, char **end, uint16 *flags)
 	lastchar = start = in;
 
 	/* Find end of word */
-	while (*in && !t_isspace(in))
+	while (*in && !isspace((unsigned char) *in))
 	{
 		lastchar = in;
 		in += pg_mblen(in);

src/backend/tsearch/dict_thesaurus.c

@@ -190,7 +190,7 @@ thesaurusRead(const char *filename, DictThesaurus *d)
 		ptr = line;
 
 		/* is it a comment? */
-		while (*ptr && t_isspace(ptr))
+		while (*ptr && isspace((unsigned char) *ptr))
 			ptr += pg_mblen(ptr);
 
 		if (t_iseq(ptr, '#') || *ptr == '\0' ||
@@ -212,7 +212,7 @@ thesaurusRead(const char *filename, DictThesaurus *d)
 								 errmsg("unexpected delimiter")));
 					state = TR_WAITSUBS;
 				}
-				else if (!t_isspace(ptr))
+				else if (!isspace((unsigned char) *ptr))
 				{
 					beginwrd = ptr;
 					state = TR_INLEX;
@@ -225,7 +225,7 @@ thesaurusRead(const char *filename, DictThesaurus *d)
 					newLexeme(d, beginwrd, ptr, idsubst, posinsubst++);
 					state = TR_WAITSUBS;
 				}
-				else if (t_isspace(ptr))
+				else if (isspace((unsigned char) *ptr))
 				{
 					newLexeme(d, beginwrd, ptr, idsubst, posinsubst++);
 					state = TR_WAITLEX;
@@ -245,7 +245,7 @@ thesaurusRead(const char *filename, DictThesaurus *d)
 					state = TR_INSUBS;
 					beginwrd = ptr + pg_mblen(ptr);
 				}
-				else if (!t_isspace(ptr))
+				else if (!isspace((unsigned char) *ptr))
 				{
 					useasis = false;
 					beginwrd = ptr;
@@ -254,7 +254,7 @@ thesaurusRead(const char *filename, DictThesaurus *d)
 			}
 			else if (state == TR_INSUBS)
 			{
-				if (t_isspace(ptr))
+				if (isspace((unsigned char) *ptr))
 				{
 					if (ptr == beginwrd)
 						ereport(ERROR,

src/backend/tsearch/spell.c

@@ -390,7 +390,7 @@ getNextFlagFromString(IspellDict *Conf, const char **sflagset, char *sflag)
 				*sflagset = next;
 				while (**sflagset)
 				{
-					if (t_isdigit(*sflagset))
+					if (isdigit((unsigned char) **sflagset))
 					{
 						if (!met_comma)
 							ereport(ERROR,
@@ -408,7 +408,7 @@ getNextFlagFromString(IspellDict *Conf, const char **sflagset, char *sflag)
 											*sflagset)));
 						met_comma = true;
 					}
-					else if (!t_isspace(*sflagset))
+					else if (!isspace((unsigned char) **sflagset))
 					{
 						ereport(ERROR,
 								(errcode(ERRCODE_CONFIG_FILE_ERROR),
@@ -542,7 +542,7 @@ NIImportDictionary(IspellDict *Conf, const char *filename)
 			while (*s)
 			{
 				/* we allow only single encoded flags for faster works */
-				if (pg_mblen(s) == 1 && t_isprint(s) && !t_isspace(s))
+				if (pg_mblen(s) == 1 && isprint((unsigned char) *s) && !isspace((unsigned char) *s))
 					s++;
 				else
 				{
@@ -558,7 +558,7 @@ NIImportDictionary(IspellDict *Conf, const char *filename)
 		s = line;
 		while (*s)
 		{
-			if (t_isspace(s))
+			if (isspace((unsigned char) *s))
 			{
 				*s = '\0';
 				break;
@@ -799,7 +799,7 @@ get_nextfield(char **str, char *next)
 		{
 			if (t_iseq(*str, '#'))
 				return false;
-			else if (!t_isspace(*str))
+			else if (!isspace((unsigned char) **str))
 			{
 				int			clen = pg_mblen(*str);
 
@@ -814,7 +814,7 @@ get_nextfield(char **str, char *next)
 		}
 		else					/* state == PAE_INMASK */
 		{
-			if (t_isspace(*str))
+			if (isspace((unsigned char) **str))
 			{
 				*next = '\0';
 				return true;
@@ -925,7 +925,7 @@ parse_affentry(char *str, char *mask, char *find, char *repl)
 		{
 			if (t_iseq(str, '#'))
 				return false;
-			else if (!t_isspace(str))
+			else if (!isspace((unsigned char) *str))
 			{
 				COPYCHAR(pmask, str);
 				pmask += pg_mblen(str);
@@ -939,7 +939,7 @@ parse_affentry(char *str, char *mask, char *find, char *repl)
 				*pmask = '\0';
 				state = PAE_WAIT_FIND;
 			}
-			else if (!t_isspace(str))
+			else if (!isspace((unsigned char) *str))
 			{
 				COPYCHAR(pmask, str);
 				pmask += pg_mblen(str);
@@ -957,7 +957,7 @@ parse_affentry(char *str, char *mask, char *find, char *repl)
 				prepl += pg_mblen(str);
 				state = PAE_INREPL;
 			}
-			else if (!t_isspace(str))
+			else if (!isspace((unsigned char) *str))
 				ereport(ERROR,
 						(errcode(ERRCODE_CONFIG_FILE_ERROR),
 						 errmsg("syntax error")));
@@ -974,7 +974,7 @@ parse_affentry(char *str, char *mask, char *find, char *repl)
 				COPYCHAR(pfind, str);
 				pfind += pg_mblen(str);
 			}
-			else if (!t_isspace(str))
+			else if (!isspace((unsigned char) *str))
 				ereport(ERROR,
 						(errcode(ERRCODE_CONFIG_FILE_ERROR),
 						 errmsg("syntax error")));
@@ -991,7 +991,7 @@ parse_affentry(char *str, char *mask, char *find, char *repl)
 				prepl += pg_mblen(str);
 				state = PAE_INREPL;
 			}
-			else if (!t_isspace(str))
+			else if (!isspace((unsigned char) *str))
 				ereport(ERROR,
 						(errcode(ERRCODE_CONFIG_FILE_ERROR),
 						 errmsg("syntax error")));
@@ -1008,7 +1008,7 @@ parse_affentry(char *str, char *mask, char *find, char *repl)
 				COPYCHAR(prepl, str);
 				prepl += pg_mblen(str);
 			}
-			else if (!t_isspace(str))
+			else if (!isspace((unsigned char) *str))
 				ereport(ERROR,
 						(errcode(ERRCODE_CONFIG_FILE_ERROR),
 						 errmsg("syntax error")));
@@ -1070,7 +1070,7 @@ addCompoundAffixFlagValue(IspellDict *Conf, char *s, uint32 val)
 	char	   *sflag;
 	int			clen;
 
-	while (*s && t_isspace(s))
+	while (*s && isspace((unsigned char) *s))
 		s += pg_mblen(s);
 
 	if (!*s)
@@ -1080,7 +1080,7 @@ addCompoundAffixFlagValue(IspellDict *Conf, char *s, uint32 val)
 
 	/* Get flag without \n */
 	sflag = sbuf;
-	while (*s && !t_isspace(s) && *s != '\n')
+	while (*s && !isspace((unsigned char) *s) && *s != '\n')
 	{
 		clen = pg_mblen(s);
 		COPYCHAR(sflag, s);
@@ -1225,7 +1225,7 @@ NIImportOOAffixes(IspellDict *Conf, const char *filename)
 
 	while ((recoded = tsearch_readline(&trst)) != NULL)
 	{
-		if (*recoded == '\0' || t_isspace(recoded) || t_iseq(recoded, '#'))
+		if (*recoded == '\0' || isspace((unsigned char) *recoded) || t_iseq(recoded, '#'))
 		{
 			pfree(recoded);
 			continue;
@@ -1262,7 +1262,7 @@ NIImportOOAffixes(IspellDict *Conf, const char *filename)
 		{
 			char	   *s = recoded + strlen("FLAG");
 
-			while (*s && t_isspace(s))
+			while (*s && isspace((unsigned char) *s))
 				s += pg_mblen(s);
 
 			if (*s)
@@ -1298,7 +1298,7 @@ NIImportOOAffixes(IspellDict *Conf, const char *filename)
 	{
 		int			fields_read;
 
-		if (*recoded == '\0' || t_isspace(recoded) || t_iseq(recoded, '#'))
+		if (*recoded == '\0' || isspace((unsigned char) *recoded) || t_iseq(recoded, '#'))
 			goto nextline;
 
 		fields_read = parse_ooaffentry(recoded, type, sflag, find, repl, mask);
@@ -1461,9 +1461,9 @@ NIImportAffixes(IspellDict *Conf, const char *filename)
 			s = findchar2(recoded, 'l', 'L');
 			if (s)
 			{
-				while (*s && !t_isspace(s))
+				while (*s && !isspace((unsigned char) *s))
 					s += pg_mblen(s);
-				while (*s && t_isspace(s))
+				while (*s && isspace((unsigned char) *s))
 					s += pg_mblen(s);
 
 				if (*s && pg_mblen(s) == 1)
@@ -1494,7 +1494,7 @@ NIImportAffixes(IspellDict *Conf, const char *filename)
 			s = recoded + 4;	/* we need non-lowercased string */
 			flagflags = 0;
 
-			while (*s && t_isspace(s))
+			while (*s && isspace((unsigned char) *s))
 				s += pg_mblen(s);
 
 			if (*s == '*')
@@ -1523,7 +1523,7 @@ NIImportAffixes(IspellDict *Conf, const char *filename)
 
 				s++;
 				if (*s == '\0' || *s == '#' || *s == '\n' || *s == ':' ||
-					t_isspace(s))
+					isspace((unsigned char) *s))
 				{
 					oldformat = true;
 					goto nextline;
@@ -1750,7 +1750,7 @@ NISortDictionary(IspellDict *Conf)
 							(errcode(ERRCODE_CONFIG_FILE_ERROR),
 							 errmsg("invalid affix alias \"%s\"",
 									Conf->Spell[i]->p.flag)));
-				if (*end != '\0' && !t_isdigit(end) && !t_isspace(end))
+				if (*end != '\0' && !isdigit((unsigned char) *end) && !isspace((unsigned char) *end))
 					ereport(ERROR,
 							(errcode(ERRCODE_CONFIG_FILE_ERROR),
 							 errmsg("invalid affix alias \"%s\"",

src/backend/tsearch/ts_locale.c

@@ -31,36 +31,6 @@ static void tsearch_readline_callback(void *arg);
  */
 #define WC_BUF_LEN  3
 
-int
-t_isdigit(const char *ptr)
-{
-	int			clen = pg_mblen(ptr);
-	wchar_t		character[WC_BUF_LEN];
-	pg_locale_t mylocale = 0;	/* TODO */
-
-	if (clen == 1 || database_ctype_is_c)
-		return isdigit(TOUCHAR(ptr));
-
-	char2wchar(character, WC_BUF_LEN, ptr, clen, mylocale);
-
-	return iswdigit((wint_t) character[0]);
-}
-
-int
-t_isspace(const char *ptr)
-{
-	int			clen = pg_mblen(ptr);
-	wchar_t		character[WC_BUF_LEN];
-	pg_locale_t mylocale = 0;	/* TODO */
-
-	if (clen == 1 || database_ctype_is_c)
-		return isspace(TOUCHAR(ptr));
-
-	char2wchar(character, WC_BUF_LEN, ptr, clen, mylocale);
-
-	return iswspace((wint_t) character[0]);
-}
-
 int
 t_isalpha(const char *ptr)
 {
@@ -91,21 +61,6 @@ t_isalnum(const char *ptr)
 	return iswalnum((wint_t) character[0]);
 }
 
-int
-t_isprint(const char *ptr)
-{
-	int			clen = pg_mblen(ptr);
-	wchar_t		character[WC_BUF_LEN];
-	pg_locale_t mylocale = 0;	/* TODO */
-
-	if (clen == 1 || database_ctype_is_c)
-		return isprint(TOUCHAR(ptr));
-
-	char2wchar(character, WC_BUF_LEN, ptr, clen, mylocale);
-
-	return iswprint((wint_t) character[0]);
-}
-
 
 /*
  * Set up to read a file using tsearch_readline().  This facility is

src/backend/tsearch/ts_utils.c

@@ -88,7 +88,7 @@ readstoplist(const char *fname, StopList *s, char *(*wordop) (const char *))
 			char	   *pbuf = line;
 
 			/* Trim trailing space */
-			while (*pbuf && !t_isspace(pbuf))
+			while (*pbuf && !isspace((unsigned char) *pbuf))
 				pbuf += pg_mblen(pbuf);
 			*pbuf = '\0';
 

src/backend/utils/adt/tsquery.c

@@ -197,7 +197,7 @@ parse_phrase_operator(TSQueryParserState pstate, int16 *distance)
 					continue;
 				}
 
-				if (!t_isdigit(ptr))
+				if (!isdigit((unsigned char) *ptr))
 					return false;
 
 				errno = 0;
@@ -274,7 +274,7 @@ parse_or_operator(TSQueryParserState pstate)
 		 * So we still treat OR literal as operation with possibly incorrect
 		 * operand and will not search it as lexeme
 		 */
-		if (!t_isspace(ptr))
+		if (!isspace((unsigned char) *ptr))
 			break;
 	}
 
@@ -315,7 +315,7 @@ gettoken_query_standard(TSQueryParserState state, int8 *operator,
 					/* generic syntax error message is fine */
 					return PT_ERR;
 				}
-				else if (!t_isspace(state->buf))
+				else if (!isspace((unsigned char) *state->buf))
 				{
 					/*
 					 * We rely on the tsvector parser to parse the value for
@@ -383,7 +383,7 @@ gettoken_query_standard(TSQueryParserState state, int8 *operator,
 				{
 					return (state->count) ? PT_ERR : PT_END;
 				}
-				else if (!t_isspace(state->buf))
+				else if (!isspace((unsigned char) *state->buf))
 				{
 					return PT_ERR;
 				}
@@ -444,7 +444,7 @@ gettoken_query_websearch(TSQueryParserState state, int8 *operator,
 					state->state = WAITOPERAND;
 					continue;
 				}
-				else if (!t_isspace(state->buf))
+				else if (!isspace((unsigned char) *state->buf))
 				{
 					/*
 					 * We rely on the tsvector parser to parse the value for
@@ -492,7 +492,7 @@ gettoken_query_websearch(TSQueryParserState state, int8 *operator,
 					state->buf++;
 					continue;
 				}
-				else if (!t_isspace(state->buf))
+				else if (!isspace((unsigned char) *state->buf))
 				{
 					/* insert implicit AND between operands */
 					state->state = WAITOPERAND;

src/backend/utils/adt/tsvector_parser.c

@@ -206,7 +206,7 @@ gettoken_tsvector(TSVectorParseState state,
 			else if ((state->oprisdelim && ISOPERATOR(state->prsbuf)) ||
 					 (state->is_web && t_iseq(state->prsbuf, '"')))
 				PRSSYNTAXERROR;
-			else if (!t_isspace(state->prsbuf))
+			else if (!isspace((unsigned char) *state->prsbuf))
 			{
 				COPYCHAR(curpos, state->prsbuf);
 				curpos += pg_mblen(state->prsbuf);
@@ -236,7 +236,7 @@ gettoken_tsvector(TSVectorParseState state,
 				statecode = WAITNEXTCHAR;
 				oldstate = WAITENDWORD;
 			}
-			else if (t_isspace(state->prsbuf) || *(state->prsbuf) == '\0' ||
+			else if (isspace((unsigned char) *state->prsbuf) || *(state->prsbuf) == '\0' ||
 					 (state->oprisdelim && ISOPERATOR(state->prsbuf)) ||
 					 (state->is_web && t_iseq(state->prsbuf, '"')))
 			{
@@ -317,7 +317,7 @@ gettoken_tsvector(TSVectorParseState state,
 		}
 		else if (statecode == INPOSINFO)
 		{
-			if (t_isdigit(state->prsbuf))
+			if (isdigit((unsigned char) *state->prsbuf))
 			{
 				if (posalen == 0)
 				{
@@ -372,10 +372,10 @@ gettoken_tsvector(TSVectorParseState state,
 					PRSSYNTAXERROR;
 				WEP_SETWEIGHT(pos[npos - 1], 0);
 			}
-			else if (t_isspace(state->prsbuf) ||
+			else if (isspace((unsigned char) *state->prsbuf) ||
 					 *(state->prsbuf) == '\0')
 				RETURN_TOKEN;
-			else if (!t_isdigit(state->prsbuf))
+			else if (!isdigit((unsigned char) *state->prsbuf))
 				PRSSYNTAXERROR;
 		}
 		else					/* internal error */

src/include/tsearch/ts_locale.h

@@ -39,11 +39,8 @@ typedef struct
 
 #define COPYCHAR(d,s)	memcpy(d, s, pg_mblen(s))
 
-extern int	t_isdigit(const char *ptr);
-extern int	t_isspace(const char *ptr);
 extern int	t_isalpha(const char *ptr);
 extern int	t_isalnum(const char *ptr);
-extern int	t_isprint(const char *ptr);
 
 extern char *lowerstr(const char *str);
 extern char *lowerstr_with_len(const char *str, int len);