Planner failed to be smart about binary-compatible expressions in pathkeys

and hash bucket-size estimation.  Issue has been there awhile but is more
critical in 7.4 because it affects varchar columns.  Per report from
Greg Stark.

src/backend/optimizer/path/costsize.c

@@ -49,7 +49,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/optimizer/path/costsize.c,v 1.116 2003/11/29 19:51:50 pgsql Exp $
+ *	  $PostgreSQL: pgsql/src/backend/optimizer/path/costsize.c,v 1.117 2003/12/03 17:45:07 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1322,6 +1322,10 @@ estimate_hash_bucketsize(Query *root, Var *var, int nbuckets)
 	float4	   *numbers;
 	int			nnumbers;
 
+	/* Ignore any binary-compatible relabeling */
+	if (var && IsA(var, RelabelType))
+		var = (Var *) ((RelabelType *) var)->arg;
+
 	/*
 	 * Lookup info about var's relation and attribute; if none available,
 	 * return default estimate.

src/backend/optimizer/path/pathkeys.c

@@ -11,7 +11,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/optimizer/path/pathkeys.c,v 1.54 2003/11/29 19:51:50 pgsql Exp $
+ *	  $PostgreSQL: pgsql/src/backend/optimizer/path/pathkeys.c,v 1.55 2003/12/03 17:45:07 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -25,12 +25,13 @@
 #include "optimizer/tlist.h"
 #include "optimizer/var.h"
 #include "parser/parsetree.h"
+#include "parser/parse_expr.h"
 #include "parser/parse_func.h"
 #include "utils/lsyscache.h"
 #include "utils/memutils.h"
 
 
-static PathKeyItem *makePathKeyItem(Node *key, Oid sortop);
+static PathKeyItem *makePathKeyItem(Node *key, Oid sortop, bool checkType);
 static List *make_canonical_pathkey(Query *root, PathKeyItem *item);
 static Var *find_indexkey_var(Query *root, RelOptInfo *rel,
 				  AttrNumber varattno);
@@ -41,10 +42,29 @@ static Var *find_indexkey_var(Query *root, RelOptInfo *rel,
  *		create a PathKeyItem node
  */
 static PathKeyItem *
-makePathKeyItem(Node *key, Oid sortop)
+makePathKeyItem(Node *key, Oid sortop, bool checkType)
 {
 	PathKeyItem *item = makeNode(PathKeyItem);
 
+	/*
+	 * Some callers pass expressions that are not necessarily of the same
+	 * type as the sort operator expects as input (for example when dealing
+	 * with an index that uses binary-compatible operators).  We must relabel
+	 * these with the correct type so that the key expressions will be seen
+	 * as equal() to expressions that have been correctly labeled.
+	 */
+	if (checkType)
+	{
+		Oid			lefttype,
+					righttype;
+
+		op_input_types(sortop, &lefttype, &righttype);
+		if (exprType(key) != lefttype)
+			key = (Node *) makeRelabelType((Expr *) key,
+										   lefttype, -1,
+										   COERCE_DONTCARE);
+	}
+
 	item->key = key;
 	item->sortop = sortop;
 	return item;
@@ -70,9 +90,11 @@ add_equijoined_keys(Query *root, RestrictInfo *restrictinfo)
 {
 	Expr	   *clause = restrictinfo->clause;
 	PathKeyItem *item1 = makePathKeyItem(get_leftop(clause),
-										 restrictinfo->left_sortop);
+										 restrictinfo->left_sortop,
+										 false);
 	PathKeyItem *item2 = makePathKeyItem(get_rightop(clause),
-										 restrictinfo->right_sortop);
+										 restrictinfo->right_sortop,
+										 false);
 	List	   *newset,
 			   *cursetlink;
 
@@ -668,7 +690,7 @@ build_index_pathkeys(Query *root,
 		}
 
 		/* OK, make a sublist for this sort key */
-		item = makePathKeyItem(indexkey, sortop);
+		item = makePathKeyItem(indexkey, sortop, true);
 		cpathkey = make_canonical_pathkey(root, item);
 
 		/*
@@ -785,7 +807,8 @@ build_subquery_pathkeys(Query *root, RelOptInfo *rel, Query *subquery)
 										tle->resdom->restypmod,
 										0);
 					outer_item = makePathKeyItem((Node *) outer_var,
-												 sub_item->sortop);
+												 sub_item->sortop,
+												 true);
 					/* score = # of mergejoin peers */
 					score = count_canonical_peers(root, outer_item);
 					/* +1 if it matches the proper query_pathkeys item */
@@ -893,7 +916,7 @@ make_pathkeys_for_sortclauses(List *sortclauses,
 		PathKeyItem *pathkey;
 
 		sortkey = get_sortgroupclause_expr(sortcl, tlist);
-		pathkey = makePathKeyItem(sortkey, sortcl->sortop);
+		pathkey = makePathKeyItem(sortkey, sortcl->sortop, true);
 
 		/*
 		 * The pathkey becomes a one-element sublist, for now;
@@ -937,7 +960,7 @@ cache_mergeclause_pathkeys(Query *root, RestrictInfo *restrictinfo)
 	{
 		oldcontext = MemoryContextSwitchTo(GetMemoryChunkContext(restrictinfo));
 		key = get_leftop(restrictinfo->clause);
-		item = makePathKeyItem(key, restrictinfo->left_sortop);
+		item = makePathKeyItem(key, restrictinfo->left_sortop, false);
 		restrictinfo->left_pathkey = make_canonical_pathkey(root, item);
 		MemoryContextSwitchTo(oldcontext);
 	}
@@ -945,7 +968,7 @@ cache_mergeclause_pathkeys(Query *root, RestrictInfo *restrictinfo)
 	{
 		oldcontext = MemoryContextSwitchTo(GetMemoryChunkContext(restrictinfo));
 		key = get_rightop(restrictinfo->clause);
-		item = makePathKeyItem(key, restrictinfo->right_sortop);
+		item = makePathKeyItem(key, restrictinfo->right_sortop, false);
 		restrictinfo->right_pathkey = make_canonical_pathkey(root, item);
 		MemoryContextSwitchTo(oldcontext);
 	}

src/backend/utils/cache/lsyscache.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/utils/cache/lsyscache.c,v 1.111 2003/11/29 19:52:00 pgsql Exp $
+ *	  $PostgreSQL: pgsql/src/backend/utils/cache/lsyscache.c,v 1.112 2003/12/03 17:45:09 tgl Exp $
  *
  * NOTES
  *	  Eventually, the index information should go through here, too.
@@ -465,6 +465,29 @@ get_opname(Oid opno)
 		return NULL;
 }
 
+/*
+ * op_input_types
+ *
+ *		Returns the left and right input datatypes for an operator
+ *		(InvalidOid if not relevant).
+ */
+void
+op_input_types(Oid opno, Oid *lefttype, Oid *righttype)
+{
+	HeapTuple	tp;
+	Form_pg_operator optup;
+
+	tp = SearchSysCache(OPEROID,
+						ObjectIdGetDatum(opno),
+						0, 0, 0);
+	if (!HeapTupleIsValid(tp))	/* shouldn't happen */
+		elog(ERROR, "cache lookup failed for operator %u", opno);
+	optup = (Form_pg_operator) GETSTRUCT(tp);
+	*lefttype = optup->oprleft;
+	*righttype = optup->oprright;
+	ReleaseSysCache(tp);
+}
+
 /*
  * op_mergejoinable
  *

src/include/utils/lsyscache.h

@@ -6,7 +6,7 @@
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/utils/lsyscache.h,v 1.85 2003/11/29 22:41:15 pgsql Exp $
+ * $PostgreSQL: pgsql/src/include/utils/lsyscache.h,v 1.86 2003/12/03 17:45:10 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -42,6 +42,7 @@ extern bool opclass_is_btree(Oid opclass);
 extern bool opclass_is_hash(Oid opclass);
 extern RegProcedure get_opcode(Oid opno);
 extern char *get_opname(Oid opno);
+extern void op_input_types(Oid opno, Oid *lefttype, Oid *righttype);
 extern bool op_mergejoinable(Oid opno, Oid *leftOp, Oid *rightOp);
 extern void op_mergejoin_crossops(Oid opno, Oid *ltop, Oid *gtop,
 					  RegProcedure *ltproc, RegProcedure *gtproc);