From 89e28d00b0ac675579e1e30d486a1128201ea6d5 Mon Sep 17 00:00:00 2001 From: "dkatz@damien-katzs-computer.local" <> Date: Thu, 17 May 2007 20:45:33 -0400 Subject: [PATCH] Bug #27119 server crash with integer division by zero during filesort on huge result Fixed a problem and compiler warning on 64bit platforms so that they only allocated UINT_MAX number of BUFFPEKS. --- sql/filesort.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/filesort.cc b/sql/filesort.cc index 16fc845f5d2..a4bf04a6786 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -370,7 +370,7 @@ static BUFFPEK *read_buffpek_from_file(IO_CACHE *buffpek_pointers, uint count) ulong length; BUFFPEK *tmp; DBUG_ENTER("read_buffpek_from_file"); - if ((ulong)count > ULONG_MAX/sizeof(BUFFPEK)) + if (count > UINT_MAX/sizeof(BUFFPEK)) return 0; /* sizeof(BUFFPEK)*count will overflow */ tmp=(BUFFPEK*) my_malloc(length=sizeof(BUFFPEK)*count, MYF(MY_WME)); if (tmp) @@ -604,7 +604,7 @@ write_keys(SORTPARAM *param, register uchar **sort_keys, uint count, MYF(MY_WME))) goto err; /* purecov: inspected */ /* check we won't have more buffpeks than we can possibly keep in memory */ - if (my_b_tell(buffpek_pointers) + sizeof(BUFFPEK) > (ulonglong)ULONG_MAX) + if (my_b_tell(buffpek_pointers) + sizeof(BUFFPEK) > (ulonglong)UINT_MAX) goto err; buffpek.file_pos= my_b_tell(tempfile); if ((ha_rows) count > param->max_rows)