Fix multibyte issue in ltree_strncasecmp().

Previously, the API for ltree_strncasecmp() took two inputs but only
one length (that of the smaller input). It truncated the larger input
to that length, but that could break a multibyte sequence.

Change the API to be a check for prefix equality (possibly
case-insensitive) instead, which is all that's needed by the
callers. Also, provide the lengths of both inputs.

Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Reviewed-by: Peter Eisentraut <peter@eisentraut.org>
Discussion: https://postgr.es/m/5f65b85740197ba6249ea507cddf609f84a6188b.camel%40j-davis.com
Backpatch-through: 14

contrib/ltree/lquery_op.c

@@ -41,7 +41,8 @@ getlexeme(char *start, char *end, int *len)
 }
 
 bool
-compare_subnode(ltree_level *t, char *qn, int len, int (*cmpptr) (const char *, const char *, size_t), bool anyend)
+compare_subnode(ltree_level *t, char *qn, int len,
+				ltree_prefix_eq_func prefix_eq, bool anyend)
 {
 	char	   *endt = t->name + t->len;
 	char	   *endq = qn + len;
@@ -57,7 +58,7 @@ compare_subnode(ltree_level *t, char *qn, int len, int (*cmpptr) (const char *,
 		while ((tn = getlexeme(tn, endt, &lent)) != NULL)
 		{
 			if ((lent == lenq || (lent > lenq && anyend)) &&
-				(*cmpptr) (qn, tn, lenq) == 0)
+				(*prefix_eq) (qn, lenq, tn, lent))
 			{
 
 				isok = true;
@@ -74,14 +75,29 @@ compare_subnode(ltree_level *t, char *qn, int len, int (*cmpptr) (const char *,
 	return true;
 }
 
-int
+/*
-ltree_strncasecmp(const char *a, const char *b, size_t s)
+ * Check if 'a' is a prefix of 'b'.
+ */
+bool
+ltree_prefix_eq(const char *a, size_t a_sz, const char *b, size_t b_sz)
 {
-	char	   *al = str_tolower(a, s, DEFAULT_COLLATION_OID);
+	if (a_sz > b_sz)
-	char	   *bl = str_tolower(b, s, DEFAULT_COLLATION_OID);
+		return false;
-	int			res;
+	else
+		return (strncmp(a, b, a_sz) == 0);
+}
 
-	res = strncmp(al, bl, s);
+/*
+ * Case-insensitive check if 'a' is a prefix of 'b'.
+ */
+bool
+ltree_prefix_eq_ci(const char *a, size_t a_sz, const char *b, size_t b_sz)
+{
+	char	   *al = str_tolower(a, a_sz, DEFAULT_COLLATION_OID);
+	char	   *bl = str_tolower(b, b_sz, DEFAULT_COLLATION_OID);
+	bool		res;
+
+	res = (strncmp(al, bl, a_sz) == 0);
 
 	pfree(al);
 	pfree(bl);
@@ -109,19 +125,19 @@ checkLevel(lquery_level *curq, ltree_level *curt)
 
 	for (int i = 0; i < curq->numvar; i++)
 	{
-		int			(*cmpptr) (const char *, const char *, size_t);
+		ltree_prefix_eq_func prefix_eq;
 
-		cmpptr = (curvar->flag & LVAR_INCASE) ? ltree_strncasecmp : strncmp;
+		prefix_eq = (curvar->flag & LVAR_INCASE) ? ltree_prefix_eq_ci : ltree_prefix_eq;
 
 		if (curvar->flag & LVAR_SUBLEXEME)
 		{
-			if (compare_subnode(curt, curvar->name, curvar->len, cmpptr,
+			if (compare_subnode(curt, curvar->name, curvar->len, prefix_eq,
 								(curvar->flag & LVAR_ANYEND)))
 				return success;
 		}
 		else if ((curvar->len == curt->len ||
 				  (curt->len > curvar->len && (curvar->flag & LVAR_ANYEND))) &&
-				 (*cmpptr) (curvar->name, curt->name, curvar->len) == 0)
+				 (*prefix_eq) (curvar->name, curvar->len, curt->name, curt->len))
 			return success;
 
 		curvar = LVAR_NEXT(curvar);

contrib/ltree/ltree.h

@@ -157,6 +157,8 @@ typedef struct
 	char		data[FLEXIBLE_ARRAY_MEMBER];
 } ltxtquery;
 
+typedef bool (*ltree_prefix_eq_func) (const char *, size_t, const char *, size_t);
+
 #define HDRSIZEQT		MAXALIGN(VARHDRSZ + sizeof(int32))
 #define COMPUTESIZE(size,lenofoperand)	( HDRSIZEQT + (size) * sizeof(ITEM) + (lenofoperand) )
 #define LTXTQUERY_TOO_BIG(size,lenofoperand) \
@@ -208,9 +210,10 @@ bool		ltree_execute(ITEM *curitem, void *checkval,
 int			ltree_compare(const ltree *a, const ltree *b);
 bool		inner_isparent(const ltree *c, const ltree *p);
 bool		compare_subnode(ltree_level *t, char *qn, int len,
-							int (*cmpptr) (const char *, const char *, size_t), bool anyend);
+							ltree_prefix_eq_func prefix_eq, bool anyend);
 ltree	   *lca_inner(ltree **a, int len);
-int			ltree_strncasecmp(const char *a, const char *b, size_t s);
+bool		ltree_prefix_eq(const char *a, size_t a_sz, const char *b, size_t b_sz);
+bool		ltree_prefix_eq_ci(const char *a, size_t a_sz, const char *b, size_t b_sz);
 
 /* fmgr macros for ltree objects */
 #define DatumGetLtreeP(X)			((ltree *) PG_DETOAST_DATUM(X))

contrib/ltree/ltxtquery_op.c

@@ -58,19 +58,19 @@ checkcondition_str(void *checkval, ITEM *val)
 	ltree_level *level = LTREE_FIRST(((CHKVAL *) checkval)->node);
 	int			tlen = ((CHKVAL *) checkval)->node->numlevel;
 	char	   *op = ((CHKVAL *) checkval)->operand + val->distance;
-	int			(*cmpptr) (const char *, const char *, size_t);
+	ltree_prefix_eq_func prefix_eq;
 
-	cmpptr = (val->flag & LVAR_INCASE) ? ltree_strncasecmp : strncmp;
+	prefix_eq = (val->flag & LVAR_INCASE) ? ltree_prefix_eq_ci : ltree_prefix_eq;
 	while (tlen > 0)
 	{
 		if (val->flag & LVAR_SUBLEXEME)
 		{
-			if (compare_subnode(level, op, val->length, cmpptr, (val->flag & LVAR_ANYEND)))
+			if (compare_subnode(level, op, val->length, prefix_eq, (val->flag & LVAR_ANYEND)))
 				return true;
 		}
 		else if ((val->length == level->len ||
 				  (level->len > val->length && (val->flag & LVAR_ANYEND))) &&
-				 (*cmpptr) (op, level->name, val->length) == 0)
+				 (*prefix_eq) (op, val->length, level->name, level->len))
 			return true;
 
 		tlen--;