diff --git a/sql/opt_range.cc b/sql/opt_range.cc index f111ecf8999..d4dfc5a65af 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -2827,7 +2827,7 @@ double records_in_column_ranges(PARAM *param, uint idx, /* Handle cases when we don't have a valid non-empty list of range */ if (!tree) - return HA_POS_ERROR; + return DBL_MAX; if (tree->type == SEL_ARG::IMPOSSIBLE) return (0L); @@ -2847,9 +2847,9 @@ double records_in_column_ranges(PARAM *param, uint idx, max_endp= range.end_key.length? &range.end_key : NULL; rows= get_column_range_cardinality(field, min_endp, max_endp, range.range_flag); - if (HA_POS_ERROR == rows) + if (DBL_MAX == rows) { - total_rows= HA_POS_ERROR; + total_rows= DBL_MAX; break; } total_rows += rows; @@ -3083,7 +3083,7 @@ bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item **cond) else { rows= records_in_column_ranges(¶m, idx, key); - if (rows != HA_POS_ERROR) + if (rows != DBL_MAX) key->field->cond_selectivity= rows/table_records; } } diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index ad71f634265..9c84729e1c2 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -3752,7 +3752,10 @@ double get_column_avg_frequency(Field * field) using the statistical data from the table column_stats. @retval - The required estimate of the rows in the column range + - The required estimate of the rows in the column range + - If there is some kind of error, this function should return DBL_MAX (and + not HA_POS_ERROR as that is an integer constant). + */ double get_column_range_cardinality(Field *field,