Tweak sorting so that nulls appear at the front of a descending sort

(vs. at the end of a normal sort).  This ensures that explicit sorts
yield the same ordering as a btree index scan.  To be really sure that
that equivalence holds, we use the btree entries in pg_amop to decide
whether we are looking at a '<' or '>' operator.  For a sort operator
that has no btree association, we put the nulls at the front if the
operator is named '>' ... pretty grotty, but it does the right thing in
simple ASC and DESC cases, and at least there's no possibility of getting
a different answer depending on the plan type chosen.

src/include/utils/tuplesort.h

@@ -13,7 +13,7 @@
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: tuplesort.h,v 1.7 2001/05/07 00:43:26 tgl Exp $
+ * $Id: tuplesort.h,v 1.8 2001/06/02 19:01:52 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -22,6 +22,7 @@
 
 #include "access/htup.h"
 #include "access/itup.h"
+#include "fmgr.h"
 
 /* Tuplesortstate is an opaque type whose details are not known outside tuplesort.c. */
 
@@ -83,6 +84,7 @@ extern void tuplesort_restorepos(Tuplesortstate *state);
 typedef enum
 {
 	SORTFUNC_LT,				/* raw "<" operator */
+	SORTFUNC_REVLT,				/* raw "<" operator, but reverse NULLs */
 	SORTFUNC_CMP,				/* -1 / 0 / 1 three-way comparator */
 	SORTFUNC_REVCMP				/* 1 / 0 / -1 (reversed) 3-way comparator */
 } SortFunctionKind;
@@ -91,4 +93,13 @@ extern void SelectSortFunction(Oid sortOperator,
 							   RegProcedure *sortFunction,
 							   SortFunctionKind *kind);
 
+/*
+ * Apply a sort function (by now converted to fmgr lookup form)
+ * and return a 3-way comparison result.  This takes care of handling
+ * NULLs and sort ordering direction properly.
+ */
+extern int32 ApplySortFunction(FmgrInfo *sortFunction, SortFunctionKind kind,
+							   Datum datum1, bool isNull1,
+							   Datum datum2, bool isNull2);
+
 #endif	 /* TUPLESORT_H */