Optimize performance of my_bitmap

MDEV-33502 Slowdown when running nested statement with many partitions

This change was triggered to help some MariaDB users with close to
10000 bits in their bitmaps.

- Change underlaying storage to be 64 bit instead of 32bit.
  - This reduses number of loops to scan bitmaps.
  - This can cause some bitmaps to be 4 byte large.
- Ensure that all not used top-bits are always 0 (simplifes code as
  the last 64 bit storage is not a special case anymore).
- Use my_find_first_bit() to find the first set bit which is much faster
  than scanning trough things byte by byte and then bit by bit.

Other things:
- Added a bool to remember if my_bitmap_init() did allocate the bitmap
  array. my_bitmap_free() will only free arrays it did allocate.
  This allowed me to remove setting 'bitmap=0' before calling
  my_bitmap_free() for cases where the bitmap's where allocated externally.
- my_bitmap_init() sets bitmap to 0 in case of failure.
- Added 'universal' asserts to most bitmap functions.
- Change all remaining calls to bitmap_init() to my_bitmap_init().
  - To finish the change from 2014.
- Changed all usage of uint32 in my_bitmap.h to my_bitmap_map.
- Updated bitmap_copy() to handle bitmaps of different size.
- Removed const from bitmap_exists_intersection() as this caused casts
  on all usage.
- Removed not used function bitmap_set_above().
- Renamed create_last_word_mask() to create_last_bit_mask() (to match
  name changes in my_bitmap.cc)
- Extended bitmap-t with test for more bitmap functions.

mysys/my_bitmap.c

@@ -13,7 +13,9 @@
 
    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA */
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335
+   USA
+ */
 
 /*
   Handling of uchar arrays as large bitmaps.
@@ -21,9 +23,18 @@
   API limitations (or, rather asserted safety assumptions,
   to encourage correct programming)
 
-    * the internal size is a set of 32 bit words
+    * the internal storage is a set of 64 bit words
     * the number of bits specified in creation can be any number > 0
 
+  Implementation notes:
+    * MY_BITMAP includes a pointer, last_word_ptr, to the last word.
+     The implication is that if one copies bitmaps to another memory
+     location, one has to call create_last_bit_mask() on the bitmap to
+     fix the internal pointer.
+   * The not used part of a the last word should always be 0.
+     This avoids special handling of the last bitmap in several cases.
+     This is checked for most calls to bitmap functions.
+
   TODO:
   Make assembler thread safe versions of these using test-and-set instructions
 
@@ -31,95 +42,94 @@
   New version written and test program added and some changes to the interface
   was made by Mikael Ronstrom 2005, with assistance of Tomas Ulin and Mats
   Kindahl.
+  Updated to 64 bits and use my_find_first_bit() to speed up
+  bitmap_get_next_set() by Monty in 2024
 */
 
 #include "mysys_priv.h"
 #include <my_bitmap.h>
 #include <m_string.h>
 #include <my_bit.h>
+#include <my_byteorder.h>
+
+
+/* Defines to check bitmaps */
+
+#define DBUG_ASSERT_BITMAP(M) \
+  DBUG_ASSERT((M)->bitmap); \
+  DBUG_ASSERT((M)->n_bits > 0); \
+  DBUG_ASSERT((M)->last_word_ptr == (M)->bitmap + no_words_in_map(M)-1); \
+  DBUG_ASSERT((*(M)->last_word_ptr & (M)->last_bit_mask) == 0);
+
+#define DBUG_ASSERT_BITMAP_AND_BIT(M,B) \
+  DBUG_ASSERT_BITMAP(M); \
+  DBUG_ASSERT((B) < (M)->n_bits);
+
+#define DBUG_ASSERT_DIFFERENT_BITMAPS(M,N) \
+  DBUG_ASSERT_BITMAP(M); \
+  DBUG_ASSERT_BITMAP(N);
+
+#define DBUG_ASSERT_IDENTICAL_BITMAPS(M,N) \
+  DBUG_ASSERT_BITMAP(M); \
+  DBUG_ASSERT_BITMAP(N); \
+  DBUG_ASSERT((M)->n_bits == (N)->n_bits);
 
 /*
-  Create a mask with the upper 'unused' bits set and the lower 'used'
-  bits clear. The bits within each byte is stored in big-endian order.
+  Create a mask for the usable bits on the LAST my_bitmap_map position for
+  a bitmap with 'bits' number of bits.
+
+  The lowest 'bits' bits are set to zero and the rest bits are set to 1.
+  For (bits & 63) == 0 , 0 is returned as in this case all bits in the
+  my_bitmap_position are significant. (This example assumes the
+  storage is ulonglong).
+
+  For 'bits & 63' it will return values from the series
+  0, 0xfffffffffffffffe,.... 0x8000000000000000
 */
 
-static inline uchar invers_last_byte_mask(uint bits)
+static inline my_bitmap_map last_bit_mask(uint bits)
 {
-  return last_byte_mask(bits) ^ 255;
+  uint bits_in_last_map= (bits & (my_bitmap_map_bits-1));
+  return bits_in_last_map ? ~((1ULL << bits_in_last_map)-1) : 0ULL;
 }
 
 
-void create_last_word_mask(MY_BITMAP *map)
+/*
+  Get a mask of the bits that are to be considered as 'on' at location
+  starting with 'bits'.
+  This function has _inv in it's name as it's usage is invers compared
+  to last_bit_mask().
+
+  For (bits & 63) it will return values from the series
+  0xffffffffffffffff, 0xfffffffffffffffe,.... 0x8000000000000000
+*/
+
+static inline my_bitmap_map first_bit_mask_inv(uint bits)
 {
-  unsigned char const mask= invers_last_byte_mask(map->n_bits);
-
-  /*
-    The first bytes are to be set to zero since they represent real  bits
-    in the bitvector. The last bytes are set to 0xFF since they  represent
-    bytes not used by the bitvector. Finally the last byte contains  bits
-    as set by the mask above.
-  */
-  unsigned char *ptr= (unsigned char*)&map->last_word_mask;
-
-  map->last_word_ptr= map->bitmap + no_words_in_map(map)-1;
-  switch (no_bytes_in_map(map) & 3) {
-  case 1:
-    map->last_word_mask= ~0U;
-    ptr[0]= mask;
-    return;
-  case 2:
-    map->last_word_mask= ~0U;
-    ptr[0]= 0;
-    ptr[1]= mask;
-    return;
-  case 3:
-    map->last_word_mask= 0U;
-    ptr[2]= mask;
-    ptr[3]= 0xFFU;
-    return;
-  case 0:
-    map->last_word_mask= 0U;
-    ptr[3]= mask;
-    return;
-  }
+  uint bits_in_last_map= (bits & (my_bitmap_map_bits-1));
+  return ~((1ULL << bits_in_last_map)-1);
 }
 
 
-static inline my_bitmap_map last_word_mask(uint bit)
+/*
+  Update the bitmap's last_word_ptr and last_bit_mask
+  Also ensure that the last world is all zero to make it
+  easy to find the next set bit.
+
+  Note that if n_bits is 0, then last_word_ptr will point to
+  bitmap (safely). The bitmap will not be usable for almost any operation.
+*/
+
+void create_last_bit_mask(MY_BITMAP *map)
 {
-  my_bitmap_map last_word_mask;
-  uint n_bits= bit + 1;
-  unsigned char const mask= invers_last_byte_mask(n_bits);
-
-  /*
-    The first bytes are to be set to zero since they represent real  bits
-    in the bitvector. The last bytes are set to 0xFF since they  represent
-    bytes not used by the bitvector. Finally the last byte contains  bits
-    as set by the mask above.
-  */
-  unsigned char *ptr= (unsigned char*)&last_word_mask;
-
-  switch ((n_bits + 7)/8 & 3) {
-  case 1:
-    last_word_mask= ~0U;
-    ptr[0]= mask;
-    break;
-  case 2:
-    last_word_mask= ~0U;
-    ptr[0]= 0;
-    ptr[1]= mask;
-    break;
-  case 3:
-    last_word_mask= 0U;
-    ptr[2]= mask;
-    ptr[3]= 0xFFU;
-    break;
-  case 0:
-    last_word_mask= 0U;
-    ptr[3]= mask;
-    break;
+  my_bitmap_map mask= last_bit_mask(map->n_bits);
+  map->last_bit_mask= mask;
+  map->last_word_ptr= map->bitmap + MY_MAX(no_words_in_map(map),1) -1;
+  if (map->n_bits > 0)
+  {
+    *map->last_word_ptr&= ~mask;           /* Set not used bits to 0 */
+    DBUG_ASSERT_BITMAP(map);
   }
-  return last_word_mask;
 }
 
 
@@ -136,25 +146,6 @@ static inline void bitmap_unlock(MY_BITMAP *map __attribute__((unused)))
 }
 
 
-static inline uint get_first_set(my_bitmap_map value, uint word_pos)
-{
-  uchar *byte_ptr= (uchar*)&value;
-  uchar byte_value;
-  uint byte_pos, bit_pos;
-
-  DBUG_ASSERT(value);
-  for (byte_pos=0; ; byte_pos++, byte_ptr++)
-  {
-    if ((byte_value= *byte_ptr))
-    {
-      for (bit_pos=0; ; bit_pos++)
-        if (byte_value & (1 << bit_pos))
-          return (word_pos*32) + (byte_pos*8) + bit_pos;
-    }
-  }
-  return MY_BIT_NONE;                           /* Impossible */
-}
-
 /*
   Initialize a bitmap object. All bits will be set to zero
 */
@@ -164,6 +155,7 @@ my_bool my_bitmap_init(MY_BITMAP *map, my_bitmap_map *buf, uint n_bits,
 {
   DBUG_ENTER("my_bitmap_init");
   map->mutex= 0;
+
   if (!buf)
   {
     uint size_in_bytes= bitmap_buffer_size(n_bits);
@@ -175,21 +167,26 @@ my_bool my_bitmap_init(MY_BITMAP *map, my_bitmap_map *buf, uint n_bits,
     }
     if (!(buf= (my_bitmap_map*) my_malloc(key_memory_MY_BITMAP_bitmap,
                                           size_in_bytes+extra, MYF(MY_WME))))
+    {
+      map->bitmap= 0;
       DBUG_RETURN(1);
+    }
     if (thread_safe)
     {
       map->mutex= (mysql_mutex_t *) ((char*) buf + size_in_bytes);
       mysql_mutex_init(key_BITMAP_mutex, map->mutex, MY_MUTEX_INIT_FAST);
     }
+    map->bitmap_allocated= 1;
   }
   else
   {
     DBUG_ASSERT(thread_safe == 0);
+    map->bitmap_allocated= 0;
   }
 
   map->bitmap= buf;
   map->n_bits= n_bits;
-  create_last_word_mask(map);
+  create_last_bit_mask(map);
   bitmap_clear_all(map);
   DBUG_RETURN(0);
 }
@@ -202,7 +199,8 @@ void my_bitmap_free(MY_BITMAP *map)
   {
     if (map->mutex)
       mysql_mutex_destroy(map->mutex);
-    my_free(map->bitmap);
+    if (map->bitmap_allocated)
+      my_free(map->bitmap);
     map->bitmap=0;
   }
   DBUG_VOID_RETURN;
@@ -227,6 +225,7 @@ my_bool bitmap_fast_test_and_set(MY_BITMAP *map, uint bitmap_bit)
   uchar *value= ((uchar*) map->bitmap) + (bitmap_bit / 8);
   uchar bit= 1 << ((bitmap_bit) & 7);
   uchar res= (*value) & bit;
+  DBUG_ASSERT_BITMAP_AND_BIT(map, bitmap_bit);
   *value|= bit;
   return res;
 }
@@ -248,8 +247,7 @@ my_bool bitmap_fast_test_and_set(MY_BITMAP *map, uint bitmap_bit)
 my_bool bitmap_test_and_set(MY_BITMAP *map, uint bitmap_bit)
 {
   my_bool res;
-  DBUG_ASSERT(map->bitmap);
-  DBUG_ASSERT(bitmap_bit < map->n_bits);
+  DBUG_ASSERT_BITMAP_AND_BIT(map, bitmap_bit);
   bitmap_lock(map);
   res= bitmap_fast_test_and_set(map, bitmap_bit);
   bitmap_unlock(map);
@@ -274,6 +272,8 @@ my_bool bitmap_fast_test_and_clear(MY_BITMAP *map, uint bitmap_bit)
   uchar *byte= (uchar*) map->bitmap + (bitmap_bit / 8);
   uchar bit= 1 << ((bitmap_bit) & 7);
   uchar res= (*byte) & bit;
+  DBUG_ASSERT_BITMAP_AND_BIT(map, bitmap_bit);
+
   *byte&= ~bit;
   return res;
 }
@@ -282,8 +282,8 @@ my_bool bitmap_fast_test_and_clear(MY_BITMAP *map, uint bitmap_bit)
 my_bool bitmap_test_and_clear(MY_BITMAP *map, uint bitmap_bit)
 {
   my_bool res;
-  DBUG_ASSERT(map->bitmap);
-  DBUG_ASSERT(bitmap_bit < map->n_bits);
+  DBUG_ASSERT_BITMAP_AND_BIT(map, bitmap_bit);
+
   bitmap_lock(map);
   res= bitmap_fast_test_and_clear(map, bitmap_bit);
   bitmap_unlock(map);
@@ -294,8 +294,8 @@ my_bool bitmap_test_and_clear(MY_BITMAP *map, uint bitmap_bit)
 uint bitmap_set_next(MY_BITMAP *map)
 {
   uint bit_found;
-  DBUG_ASSERT(map->bitmap);
-  if ((bit_found= bitmap_get_first(map)) != MY_BIT_NONE)
+  DBUG_ASSERT_BITMAP(map);
+  if ((bit_found= bitmap_get_first_clear(map)) != MY_BIT_NONE)
     bitmap_set_bit(map, bit_found);
   return bit_found;
 }
@@ -305,15 +305,16 @@ uint bitmap_set_next(MY_BITMAP *map)
   Set the specified number of bits in the bitmap buffer.
 
   @param map         [IN]       Bitmap
-  @param prefix_size [IN]       Number of bits to be set
+  @param prefix_size [IN]       Number of bits to be set or (uint) ~0 for all
 */
+
 void bitmap_set_prefix(MY_BITMAP *map, uint prefix_size)
 {
   uint prefix_bytes, prefix_bits, d;
   uchar *m= (uchar *)map->bitmap;
-
-  DBUG_ASSERT(map->bitmap);
+  DBUG_ASSERT_BITMAP(map);
   DBUG_ASSERT(prefix_size <= map->n_bits || prefix_size == (uint) ~0);
+
   set_if_smaller(prefix_size, map->n_bits);
   if ((prefix_bytes= prefix_size / 8))
     memset(m, 0xff, prefix_bytes);
@@ -326,37 +327,45 @@ void bitmap_set_prefix(MY_BITMAP *map, uint prefix_size)
   }
   if ((d= no_bytes_in_map(map)-prefix_bytes))
     memset(m, 0, d);
+  DBUG_ASSERT_BITMAP(map);
 }
 
 
+/**
+   Check if bitmap is a bitmap of prefix bits set in the beginning
+
+   @param map          bitmap
+   @param prefix_size  number of bits that should be set. 0 is allowed.
+
+   @return 1           Yes, prefix bits where set or prefix_size == 0.
+   @return 0           No
+*/
+
 my_bool bitmap_is_prefix(const MY_BITMAP *map, uint prefix_size)
 {
   uint prefix_mask= last_byte_mask(prefix_size);
   uchar *m= (uchar*) map->bitmap;
   uchar *end_prefix= m+(prefix_size-1)/8;
   uchar *end;
-  DBUG_ASSERT(m);
-  DBUG_ASSERT(prefix_size <= map->n_bits);
 
   /* Empty prefix is always true */
   if (!prefix_size)
     return 1;
 
+  DBUG_ASSERT_BITMAP_AND_BIT(map, prefix_size-1);
+
   while (m < end_prefix)
     if (*m++ != 0xff)
       return 0;
 
-  end= ((uchar*) map->bitmap) + no_bytes_in_map(map) - 1;
-  if (m == end)
-    return ((*m & last_byte_mask(map->n_bits)) == prefix_mask);
-
+  end= ((uchar*) map->last_word_ptr) + sizeof(*map->last_word_ptr)-1;
   if (*m != prefix_mask)
     return 0;
 
-  while (++m < end)
+  while (++m <= end)
     if (*m != 0)
       return 0;
-  return ((*m & last_byte_mask(map->n_bits)) == 0);
+  return 1;
 }
 
 
@@ -364,10 +373,12 @@ my_bool bitmap_is_set_all(const MY_BITMAP *map)
 {
   my_bitmap_map *data_ptr= map->bitmap;
   my_bitmap_map *end= map->last_word_ptr;
+  DBUG_ASSERT_BITMAP(map);
+
   for (; data_ptr < end; data_ptr++)
-    if (*data_ptr != 0xFFFFFFFF)
+    if (*data_ptr != ~(my_bitmap_map)0)
       return FALSE;
-  return (*data_ptr | map->last_word_mask) == 0xFFFFFFFF;
+  return (*data_ptr | map->last_bit_mask) == ~(my_bitmap_map)0;
 }
 
 
@@ -375,61 +386,58 @@ my_bool bitmap_is_clear_all(const MY_BITMAP *map)
 {
   my_bitmap_map *data_ptr= map->bitmap;
   my_bitmap_map *end= map->last_word_ptr;
+  DBUG_ASSERT_BITMAP(map);
 
-  DBUG_ASSERT(map->n_bits > 0);
-  for (; data_ptr < end; data_ptr++)
+  for (; data_ptr <= end; data_ptr++)
     if (*data_ptr)
       return FALSE;
-  return (*data_ptr & ~map->last_word_mask) == 0;
+  return TRUE;
 }
 
+
 /* Return TRUE if map1 is a subset of map2 */
 
 my_bool bitmap_is_subset(const MY_BITMAP *map1, const MY_BITMAP *map2)
 {
-  my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end;
+  my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end= map1->last_word_ptr;
+  DBUG_ASSERT_IDENTICAL_BITMAPS(map1,map2);
 
-  DBUG_ASSERT(map1->bitmap && map2->bitmap);
-  DBUG_ASSERT(map1->n_bits==map2->n_bits);
-
-  end= map1->last_word_ptr;
-  while (m1 < end)
+  while (m1 <= end)
   {
     if ((*m1++) & ~(*m2++))
       return 0;
   }
-  /* here both maps have the same number of bits - see assert above */
-  return ((*m1 & ~*m2 & ~map1->last_word_mask) ? 0 : 1);
+  return 1;
 }
 
 /* True if bitmaps has any common bits */
 
 my_bool bitmap_is_overlapping(const MY_BITMAP *map1, const MY_BITMAP *map2)
 {
-  my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end;
+  my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end= map1->last_word_ptr;
+  DBUG_ASSERT_IDENTICAL_BITMAPS(map1,map2);
 
-  DBUG_ASSERT(map1->bitmap);
-  DBUG_ASSERT(map2->bitmap);
-  DBUG_ASSERT(map1->n_bits==map2->n_bits);
-
-  end= map1->last_word_ptr;
-  while (m1 < end)
+  while (m1 <= end)
   {
     if ((*m1++) & (*m2++))
       return 1;
   }
-  /* here both maps have the same number of bits - see assert above */
-  return ((*m1 & *m2 & ~map1->last_word_mask) ? 1 : 0);
+  return 0;
 }
 
 
+/*
+  Create intersection of two bitmaps
+
+  @param map    map1. Result is stored here
+  @param map2   map2
+*/
+
 void bitmap_intersect(MY_BITMAP *map, const MY_BITMAP *map2)
 {
   my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end;
   uint len= no_words_in_map(map), len2 = no_words_in_map(map2);
-
-  DBUG_ASSERT(map->bitmap);
-  DBUG_ASSERT(map2->bitmap);
+  DBUG_ASSERT_DIFFERENT_BITMAPS(map,map2);
 
   end= to+MY_MIN(len,len2);
   while (to < end)
@@ -437,7 +445,7 @@ void bitmap_intersect(MY_BITMAP *map, const MY_BITMAP *map2)
 
   if (len2 <= len)
   {
-    to[-1]&= ~map2->last_word_mask; /* Clear last not relevant bits */
+    to[-1]&= ~map2->last_bit_mask; /* Clear last not relevant bits */
     end+= len-len2;
     while (to < end)
       *to++= 0;
@@ -447,50 +455,51 @@ void bitmap_intersect(MY_BITMAP *map, const MY_BITMAP *map2)
 
 /*
   Check if there is some bit index between start_bit and end_bit, such that
-  this is bit is set for all bitmaps in bitmap_list.
+  this is at least on bit that set for all bitmaps in bitmap_list.
 
   SYNOPSIS
     bitmap_exists_intersection()
     bitmpap_array [in]  a set of MY_BITMAPs
-    bitmap_count  [in]  number of elements in bitmpap_array
+    bitmap_count  [in]  number of elements in bitmap_array
     start_bit     [in]  beginning (inclusive) of the range of bits to search
     end_bit       [in]  end (inclusive) of the range of bits to search, must be
                         no bigger than the bits of the shortest bitmap.
 
-  NOTES
-  This function assumes that for at least one of the bitmaps in bitmap_array all
-  bits outside the range [start_bit, end_bit] are 0. As a result is not
-  necessary to take care of the bits outside the range [start_bit, end_bit].
-
   RETURN
     TRUE  if an intersecion exists
     FALSE no intersection
 */
 
-my_bool bitmap_exists_intersection(const MY_BITMAP **bitmap_array,
+my_bool bitmap_exists_intersection(MY_BITMAP **bitmap_array,
                                    uint bitmap_count,
                                    uint start_bit, uint end_bit)
 {
   uint i, j, start_idx, end_idx;
-  my_bitmap_map cur_res;
+  my_bitmap_map cur_res, first_map;
 
   DBUG_ASSERT(bitmap_count);
   DBUG_ASSERT(end_bit >= start_bit);
   for (j= 0; j < bitmap_count; j++)
-    DBUG_ASSERT(end_bit < bitmap_array[j]->n_bits);
+  {
+    DBUG_ASSERT_BITMAP_AND_BIT(bitmap_array[j], end_bit);
+  }
 
   start_idx= start_bit/8/sizeof(my_bitmap_map);
   end_idx= end_bit/8/sizeof(my_bitmap_map);
 
+  first_map= first_bit_mask_inv(start_bit);
+  cur_res= first_map;
   for (i= start_idx; i < end_idx; i++)
   {
-    cur_res= ~0;
     for (j= 0; cur_res && j < bitmap_count; j++)
       cur_res &= bitmap_array[j]->bitmap[i];
     if (cur_res)
       return TRUE;
+    cur_res= ~(my_bitmap_map) 0;
   }
-  cur_res= ~last_word_mask(end_bit);
+  cur_res= ~last_bit_mask(end_bit+1);
+  if (start_idx == end_idx)
+    cur_res&= first_map;
   for (j= 0; cur_res && j < bitmap_count; j++)
     cur_res &= bitmap_array[j]->bitmap[end_idx];
   return cur_res != 0;
@@ -501,60 +510,21 @@ my_bool bitmap_exists_intersection(const MY_BITMAP **bitmap_array,
 
 my_bool bitmap_union_is_set_all(const MY_BITMAP *map1, const MY_BITMAP *map2)
 {
-  my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end;
+  my_bitmap_map *m1= map1->bitmap, *m2= map2->bitmap, *end= map1->last_word_ptr;
+  DBUG_ASSERT_IDENTICAL_BITMAPS(map1,map2);
 
-  DBUG_ASSERT(map1->bitmap);
-  DBUG_ASSERT(map2->bitmap);
-  DBUG_ASSERT(map1->n_bits==map2->n_bits);
-  end= map1->last_word_ptr;
   while ( m1 < end)
-    if ((*m1++ | *m2++) != 0xFFFFFFFF)
+    if ((*m1++ | *m2++) != ~(my_bitmap_map)0)
       return FALSE;
   /* here both maps have the same number of bits - see assert above */
-  return ((*m1 | *m2 | map1->last_word_mask) != 0xFFFFFFFF);
-}
-
-
-
-/*
-  Set/clear all bits above a bit.
-
-  SYNOPSIS
-    bitmap_set_above()
-    map                  RETURN The bitmap to change.
-    from_byte                   The bitmap buffer byte offset to start with.
-    use_bit                     The bit value (1/0) to use for all upper bits.
-
-  NOTE
-    You can only set/clear full bytes.
-    The function is meant for the situation that you copy a smaller bitmap
-    to a bigger bitmap. Bitmap lengths are always multiple of eigth (the
-    size of a byte). Using 'from_byte' saves multiplication and division
-    by eight during parameter passing.
-
-  RETURN
-    void
-*/
-
-void bitmap_set_above(MY_BITMAP *map, uint from_byte, uint use_bit)
-{
-  uchar use_byte= use_bit ? 0xff : 0;
-  uchar *to= (uchar *)map->bitmap + from_byte;
-  uchar *end= (uchar *)map->bitmap + (map->n_bits+7)/8;
-
-  while (to < end)
-    *to++= use_byte;
+  return ((*m1 | *m2 | map1->last_bit_mask) != ~(my_bitmap_map)0);
 }
 
 
 void bitmap_subtract(MY_BITMAP *map, const MY_BITMAP *map2)
 {
-  my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end;
-  DBUG_ASSERT(map->bitmap);
-  DBUG_ASSERT(map2->bitmap);
-  DBUG_ASSERT(map->n_bits==map2->n_bits);
-
-  end= map->last_word_ptr;
+  my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end= map->last_word_ptr;
+  DBUG_ASSERT_IDENTICAL_BITMAPS(map,map2);
 
   while (to <= end)
     *to++ &= ~(*from++);
@@ -563,12 +533,8 @@ void bitmap_subtract(MY_BITMAP *map, const MY_BITMAP *map2)
 
 void bitmap_union(MY_BITMAP *map, const MY_BITMAP *map2)
 {
-  my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end;
-
-  DBUG_ASSERT(map->bitmap);
-  DBUG_ASSERT(map2->bitmap);
-  DBUG_ASSERT(map->n_bits == map2->n_bits);
-  end= map->last_word_ptr;
+  my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end= map->last_word_ptr;
+  DBUG_ASSERT_IDENTICAL_BITMAPS(map,map2);
 
   while (to <= end)
     *to++ |= *from++;
@@ -578,9 +544,8 @@ void bitmap_union(MY_BITMAP *map, const MY_BITMAP *map2)
 void bitmap_xor(MY_BITMAP *map, const MY_BITMAP *map2)
 {
   my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end= map->last_word_ptr;
-  DBUG_ASSERT(map->bitmap);
-  DBUG_ASSERT(map2->bitmap);
-  DBUG_ASSERT(map->n_bits == map2->n_bits);
+  DBUG_ASSERT_IDENTICAL_BITMAPS(map,map2);
+
   while (to <= end)
     *to++ ^= *from++;
 }
@@ -588,13 +553,14 @@ void bitmap_xor(MY_BITMAP *map, const MY_BITMAP *map2)
 
 void bitmap_invert(MY_BITMAP *map)
 {
-  my_bitmap_map *to= map->bitmap, *end;
+  my_bitmap_map *to= map->bitmap, *end= map->last_word_ptr;
+  DBUG_ASSERT_BITMAP(map);
 
-  DBUG_ASSERT(map->bitmap);
-  end= map->last_word_ptr;
+  while (to < end)
+    *to++ ^= ~(my_bitmap_map)0;
 
-  while (to <= end)
-    *to++ ^= 0xFFFFFFFF;
+  *to ^= (~(my_bitmap_map)0 & ~map->last_bit_mask);
+  DBUG_ASSERT_BITMAP(map);
 }
 
 
@@ -603,45 +569,54 @@ uint bitmap_bits_set(const MY_BITMAP *map)
   my_bitmap_map *data_ptr= map->bitmap;
   my_bitmap_map *end= map->last_word_ptr;
   uint res= 0;
-  DBUG_ASSERT(map->bitmap);
+  DBUG_ASSERT_BITMAP(map);
 
-  for (; data_ptr < end; data_ptr++)
-    res+= my_count_bits_uint32(*data_ptr);
+  for (; data_ptr <= end; data_ptr++)
+    res+= my_count_bits(*data_ptr);
 
-  /*Reset last bits to zero*/
-  res+= my_count_bits_uint32(*map->last_word_ptr & ~map->last_word_mask);
   return res;
 }
 
-void bitmap_copy(MY_BITMAP *map, const MY_BITMAP *map2)
+
+/**
+  Copy bitmaps
+
+  @param map1   to-bitmap
+  @param map2   from-bitmap
+
+  @notes
+  Code will work even of the bitmaps are of different size.
+  In this case, only up to to->n_bits will be copied.
+*/
+
+void bitmap_copy(MY_BITMAP *map1, const MY_BITMAP *map2)
 {
-  my_bitmap_map *to= map->bitmap, *from= map2->bitmap, *end;
+  my_bitmap_map *to= map1->bitmap, *from= map2->bitmap;
+  uint map1_length= no_words_in_map(map1)*sizeof(my_bitmap_map);
+  uint map2_length= no_words_in_map(map2)*sizeof(my_bitmap_map);
+  uint length= MY_MIN(map1_length, map2_length);
+  DBUG_ASSERT_DIFFERENT_BITMAPS(map1,map2);
 
-  DBUG_ASSERT(map->bitmap);
-  DBUG_ASSERT(map2->bitmap);
-  DBUG_ASSERT(map->n_bits == map2->n_bits);
-  end= map->last_word_ptr;
-
-  while (to <= end)
-    *to++ = *from++;
+  memcpy(to, from, length);
+  if (length < map1_length)
+    bzero(to + length, map1_length - length);
+  *map1->last_word_ptr&= ~map1->last_bit_mask;
 }
 
 
+/*
+  Find first set bit in the bitmap
+*/
+
 uint bitmap_get_first_set(const MY_BITMAP *map)
 {
-  uint i;
   my_bitmap_map *data_ptr= map->bitmap, *end= map->last_word_ptr;
+  DBUG_ASSERT_BITMAP(map);
 
-  DBUG_ASSERT(map->bitmap);
-
-  for (i=0; data_ptr < end; data_ptr++, i++)
+  for (uint i=0; data_ptr <= end; data_ptr++, i++)
     if (*data_ptr)
-      goto found;
-  if (!(*data_ptr & ~map->last_word_mask))
-    return MY_BIT_NONE;
-
-found:
-  return get_first_set(*data_ptr, i);
+      return my_find_first_bit(*data_ptr) + i * sizeof(my_bitmap_map)*8;
+  return MY_BIT_NONE;
 }
 
 
@@ -656,82 +631,54 @@ found:
 
 uint bitmap_get_next_set(const MY_BITMAP *map, uint bitmap_bit)
 {
-  uint word_pos, byte_to_mask, i;
-  union { my_bitmap_map bitmap ; uchar bitmap_buff[sizeof(my_bitmap_map)]; }
-  first_word;
-  uchar *ptr= &first_word.bitmap_buff[0];
-  my_bitmap_map *data_ptr, *end= map->last_word_ptr;
-
-  DBUG_ASSERT(map->bitmap);
+  uint word_pos;
+  my_bitmap_map first_word, *data_ptr, *end= map->last_word_ptr;
+  DBUG_ASSERT_BITMAP(map);
 
   /* Look for the next bit */
   bitmap_bit++;
   if (bitmap_bit >= map->n_bits)
     return MY_BIT_NONE;
-  word_pos= bitmap_bit / 32;
+
+  word_pos= bitmap_bit / 64;
   data_ptr= map->bitmap + word_pos;
-  first_word.bitmap= *data_ptr;
 
-  /* Mask out previous bits from first_word */
-  byte_to_mask= (bitmap_bit % 32) / 8;
-  for (i= 0; i < byte_to_mask; i++)
-    ptr[i]= 0;
-  ptr[byte_to_mask]&= 0xFFU << (bitmap_bit & 7);
+  first_word= *data_ptr & first_bit_mask_inv(bitmap_bit);
 
-  if (data_ptr == end)
+  if (first_word)
   {
-    if (first_word.bitmap & ~map->last_word_mask)
-      return get_first_set(first_word.bitmap, word_pos);
-    else
-      return MY_BIT_NONE;
+    /* Optimize common case when most bits are set */
+    if (first_word & (1ULL << ((bitmap_bit & (my_bitmap_map_bits-1)))))
+      return bitmap_bit;
+    return my_find_first_bit(first_word) + (bitmap_bit & ~(my_bitmap_map_bits-1));
   }
-   
-  if (first_word.bitmap)
-    return get_first_set(first_word.bitmap, word_pos);
 
-  for (data_ptr++, word_pos++; data_ptr < end; data_ptr++, word_pos++)
+  for (data_ptr++; data_ptr <= end; data_ptr++)
+  {
+    bitmap_bit+= 64;
     if (*data_ptr)
-      return get_first_set(*data_ptr, word_pos);
-
-  if (!(*end & ~map->last_word_mask))
-    return MY_BIT_NONE;
-  return get_first_set(*end, word_pos);
+      return my_find_first_bit(*data_ptr) + (bitmap_bit & ~(my_bitmap_map_bits-1));
+  }
+  return MY_BIT_NONE;
 }
 
 
-/* Get first free bit */
+/* Get first clear bit */
 
-uint bitmap_get_first(const MY_BITMAP *map)
+uint bitmap_get_first_clear(const MY_BITMAP *map)
 {
-  uchar *byte_ptr;
-  uint i,j,k;
-  my_bitmap_map *data_ptr, *end= map->last_word_ptr;
+  uint i;
+  my_bitmap_map *data_ptr= map->bitmap, *end= map->last_word_ptr;
+  DBUG_ASSERT_BITMAP(map);
 
-  DBUG_ASSERT(map->bitmap);
-  data_ptr= map->bitmap;
-  *map->last_word_ptr|= map->last_word_mask;
-
-  for (i=0; data_ptr < end; data_ptr++, i++)
-    if (*data_ptr != 0xFFFFFFFF)
+  for (i= 0; data_ptr < end; data_ptr++, i++)
+    if (*data_ptr != ~(my_bitmap_map)0)
       goto found;
-  if ((*data_ptr | map->last_word_mask) == 0xFFFFFFFF)
+  if ((*data_ptr | map->last_bit_mask) == ~(my_bitmap_map)0)
     return MY_BIT_NONE;
-
 found:
-  byte_ptr= (uchar*)data_ptr;
-  for (j=0; ; j++, byte_ptr++)
-  {
-    if (*byte_ptr != 0xFF)
-    {
-      for (k=0; ; k++)
-      {
-        if (!(*byte_ptr & (1 << k)))
-          return (i*32) + (j*8) + k;
-      }
-    }
-  }
-  DBUG_ASSERT(0);
-  return MY_BIT_NONE;                           /* Impossible */
+  /* find first zero bit by reverting all bits and find first bit */
+  return my_find_first_bit(~*data_ptr) + i * sizeof(my_bitmap_map)*8;
 }
 
 
@@ -753,4 +700,3 @@ void bitmap_lock_clear_bit(MY_BITMAP *map, uint bitmap_bit)
   bitmap_clear_bit(map, bitmap_bit);
   bitmap_unlock(map);
 }
-