Repair crash with unsortable data in grouping sets.

Previously the code would generate incorrect results, assertion
failures, or crashes if given unsortable (but hashable) columns in
grouping sets.  Handle by throwing an error instead.

Report and patch by Pavan Deolasee (though I changed the error
wording slightly); regression test by me.

(This affects 9.5 only since the planner was refactored in 9.6.)

src/backend/optimizer/plan/planner.c

@@ -1756,6 +1756,11 @@ grouping_planner(PlannerInfo *root, double tuple_fraction)
 
 			if (parse->groupingSets)
 			{
+				if (!grouping_is_sortable(parse->groupClause))
+					ereport(ERROR,
+							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+							 errmsg("could not implement GROUP BY"),
+							 errdetail("Some of the datatypes do not support sorting, which is required for grouping sets.")));
 				use_hashed_grouping = false;
 			}
 			else

src/test/regress/expected/groupingsets.out

@@ -847,4 +847,11 @@ select sum(ten) from onek group by rollup(four::text), two order by 1;
  2500
 (6 rows)
 
+-- Check for sensible failure on unsortable data
+select usort, grouping(usort), count(*) from (values ('1'::xid),('2'::xid),('1'::xid)) v(usort) group by rollup(usort);
+ERROR:  could not implement GROUP BY
+DETAIL:  Some of the datatypes do not support sorting, which is required for grouping sets.
+select id, usort, grouping(id,usort), count(*) from (values (1,'1'::xid),(1,'2'::xid),(2,'1'::xid)) v(id,usort) group by grouping sets ((id), (usort));
+ERROR:  could not implement GROUP BY
+DETAIL:  Some of the datatypes do not support sorting, which is required for grouping sets.
 -- end

src/test/regress/sql/groupingsets.sql

@@ -224,4 +224,8 @@ select array(select row(v.a,s1.*) from (select two,four, count(*) from onek grou
 select sum(ten) from onek group by two, rollup(four::text) order by 1;
 select sum(ten) from onek group by rollup(four::text), two order by 1;
 
+-- Check for sensible failure on unsortable data
+select usort, grouping(usort), count(*) from (values ('1'::xid),('2'::xid),('1'::xid)) v(usort) group by rollup(usort);
+select id, usort, grouping(id,usort), count(*) from (values (1,'1'::xid),(1,'2'::xid),(2,'1'::xid)) v(id,usort) group by grouping sets ((id), (usort));
+
 -- end