From 1bd3a8f58bb8147628a85643b0246051a6e4e5ee Mon Sep 17 00:00:00 2001
From: Tom Lane <tgl@sss.pgh.pa.us>
Date: Thu, 3 Aug 2000 00:58:22 +0000
Subject: [PATCH] Repair bug noted by Paul Caskey: neqsel() has been generating
 a bogus result, in fact nearly the opposite of what it should, because it was
 passing the not-equal operator to eqsel() which would use it to compare the
 value against the most common value in the column, and of course obtain the
 wrong result therefrom.  Must pass the equality operator to eqsel() instead. 
 Fortunately that's easy to get from the oprnegate link.

---
 src/backend/utils/adt/selfuncs.c | 63 +++++++++++++++++++++++++++++---
 1 file changed, 58 insertions(+), 5 deletions(-)

diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c
index 04d9d9f6a9e..16b7964392e 100644
--- a/src/backend/utils/adt/selfuncs.c
+++ b/src/backend/utils/adt/selfuncs.c
@@ -15,7 +15,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.76 2000/07/29 03:26:42 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.77 2000/08/03 00:58:22 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -252,9 +252,33 @@ eqsel(PG_FUNCTION_ARGS)
 Datum
 neqsel(PG_FUNCTION_ARGS)
 {
+	Oid			opid = PG_GETARG_OID(0);
+	Oid			relid = PG_GETARG_OID(1);
+	AttrNumber	attno = PG_GETARG_INT16(2);
+	Datum		value = PG_GETARG_DATUM(3);
+	int32		flag = PG_GETARG_INT32(4);
+	Oid			eqopid;
 	float8		result;
 
-	result = DatumGetFloat8(eqsel(fcinfo));
+	/*
+	 * We want 1 - eqsel() where the equality operator is the one associated
+	 * with this != operator, that is, its negator.
+	 */
+	eqopid = get_negator(opid);
+	if (eqopid)
+	{
+		result = DatumGetFloat8(DirectFunctionCall5(eqsel,
+													ObjectIdGetDatum(eqopid),
+													ObjectIdGetDatum(relid),
+													Int16GetDatum(attno),
+													value,
+													Int32GetDatum(flag)));
+	}
+	else
+	{
+		/* Use default selectivity (should we raise an error instead?) */
+		result = DEFAULT_EQ_SEL;
+	}
 	result = 1.0 - result;
 	PG_RETURN_FLOAT8(result);
 }
@@ -392,15 +416,39 @@ scalarltsel(PG_FUNCTION_ARGS)
 Datum
 scalargtsel(PG_FUNCTION_ARGS)
 {
+	Oid			opid = PG_GETARG_OID(0);
+	Oid			relid = PG_GETARG_OID(1);
+	AttrNumber	attno = PG_GETARG_INT16(2);
+	Datum		value = PG_GETARG_DATUM(3);
+	int32		flag = PG_GETARG_INT32(4);
+	Oid			ltopid;
 	float8		result;
 
 	/*
 	 * Compute selectivity of "<", then invert --- but only if we were
-	 * able to produce a non-default estimate.
+	 * able to produce a non-default estimate.  Note that we get the
+	 * negator which strictly speaking means we are looking at "<="
+	 * for ">" or "<" for ">=".  We assume this won't matter.
 	 */
-	result = DatumGetFloat8(scalarltsel(fcinfo));
+	ltopid = get_negator(opid);
+	if (ltopid)
+	{
+		result = DatumGetFloat8(DirectFunctionCall5(scalarltsel,
+													ObjectIdGetDatum(ltopid),
+													ObjectIdGetDatum(relid),
+													Int16GetDatum(attno),
+													value,
+													Int32GetDatum(flag)));
+	}
+	else
+	{
+		/* Use default selectivity (should we raise an error instead?) */
+		result = DEFAULT_INEQ_SEL;
+	}
+
 	if (result != DEFAULT_INEQ_SEL)
 		result = 1.0 - result;
+
 	PG_RETURN_FLOAT8(result);
 }
 
@@ -567,7 +615,7 @@ nlikesel(PG_FUNCTION_ARGS)
 Datum
 eqjoinsel(PG_FUNCTION_ARGS)
 {
-#ifdef NOT_USED
+#ifdef NOT_USED					/* see neqjoinsel() before removing me! */
 	Oid			opid = PG_GETARG_OID(0);
 #endif
 	Oid			relid1 = PG_GETARG_OID(1);
@@ -620,6 +668,11 @@ neqjoinsel(PG_FUNCTION_ARGS)
 {
 	float8		result;
 
+	/*
+	 * XXX we skip looking up the negator operator here because we know
+	 * eqjoinsel() won't look at it anyway.  If eqjoinsel() ever does look,
+	 * this routine will need to look more like neqsel() does.
+	 */
 	result = DatumGetFloat8(eqjoinsel(fcinfo));
 	result = 1.0 - result;
 	PG_RETURN_FLOAT8(result);